[llvm-branch-commits] [llvm] [AMDGPU] Update calling convention for true16 mode (PR #218463)
Guo Chen via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Aug 25 08:12:33 PDT 2026
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/218463
>From 4fc585f2c90a3ee2544273ed48e86227bdaf1254 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Wed, 1 Apr 2026 11:20:27 -0400
Subject: [PATCH] calling conv update
---
llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 27 +-
llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td | 69 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 5 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll | 27 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 30 +-
.../GlobalISel/combine-fma-add-fma-mul.ll | 12 +-
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll | 19 +-
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll | 2 +-
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll | 2 +-
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll | 19 +-
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 441 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 527 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 32 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 41 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll | 19 +-
.../AMDGPU/GlobalISel/insertelement.i16.ll | 141 +-
.../AMDGPU/GlobalISel/intrinsic-trunc.ll | 2 +-
...galize-llvm.amdgcn.image.atomic.dim.a16.ll | 798 +-
.../legalize-llvm.amdgcn.image.sample.a16.ll | 2176 ++---
...galize-llvm.amdgcn.image.sample.g16.a16.ll | 204 +-
.../legalize-llvm.amdgcn.image.sample.g16.ll | 1048 +-
...legalize-llvm.amdgcn.image.store.2d.d16.ll | 8 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.fract.ll | 2 +-
.../llvm.amdgcn.image.atomic.dim.a16.ll | 40 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 135 +-
.../llvm.amdgcn.image.load.2darraymsaa.a16.ll | 25 +-
.../llvm.amdgcn.image.load.3d.a16.ll | 19 +-
.../llvm.amdgcn.image.sample.g16.ll | 18 +-
.../GlobalISel/llvm.amdgcn.interp.inreg.ll | 2 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 26 +-
.../llvm.amdgcn.raw.buffer.load.format.f16.ll | 16 +-
.../GlobalISel/llvm.amdgcn.raw.buffer.load.ll | 15 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 24 +-
.../llvm.amdgcn.raw.buffer.store.ll | 13 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 10 +-
.../llvm.amdgcn.raw.tbuffer.load.f16.ll | 36 +-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 254 +-
.../llvm.amdgcn.raw.tbuffer.store.i8.ll | 50 +-
...vm.amdgcn.struct.buffer.load.format.f16.ll | 15 +-
.../llvm.amdgcn.struct.buffer.load.ll | 10 +-
...m.amdgcn.struct.buffer.store.format.f16.ll | 37 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll | 52 -
.../llvm.amdgcn.struct.tbuffer.load.f16.ll | 67 +-
.../AMDGPU/GlobalISel/load-constant.96.ll | 22 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll | 27 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll | 27 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 8452 ++++++++---------
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 2204 +++--
.../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll | 558 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 4764 ++++++----
.../CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll | 1386 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 306 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 3217 ++++---
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 1305 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 1154 ++-
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 8 +-
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 102 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 794 +-
llvm/test/CodeGen/AMDGPU/bitop3.ll | 81 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 642 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 642 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 642 +-
.../CodeGen/AMDGPU/call-argument-types.ll | 105 +-
.../CodeGen/AMDGPU/calling-conventions.ll | 448 +-
.../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 6 +-
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 25 +-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 156 +-
.../AMDGPU/divergence-driven-buildvector.ll | 5 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 2 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 11 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 166 +-
llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fdiv.f16.ll | 8 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 1659 ++--
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 2041 ++--
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 2041 ++--
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 1637 ++--
llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll | 535 +-
.../float-to-arbitrary-fp-fp8-f16-hw.ll | 14 +-
.../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 222 +-
.../AMDGPU/float-to-arbitrary-fp-widen.ll | 542 +-
llvm/test/CodeGen/AMDGPU/fma.f16.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll | 98 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmaximum3.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 19 +-
llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll | 9 +-
llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll | 98 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fminimum3.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 19 +-
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 32 +-
.../CodeGen/AMDGPU/fneg-modifier-casting.ll | 34 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 630 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll | 54 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 586 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 62 +-
llvm/test/CodeGen/AMDGPU/function-returns.ll | 90 +-
.../AMDGPU/gfx-callable-argument-types.ll | 1260 ++-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 2162 +++--
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 2025 ++--
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 2025 ++--
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 1637 ++--
llvm/test/CodeGen/AMDGPU/global-saddr-load.ll | 32 +-
llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll | 138 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 193 +-
.../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 46 +-
.../isel-amdgpu-cs-chain-preserve-cc.ll | 26 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 8 +-
.../test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll | 14 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll | 1 +
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll | 2 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 39 +-
.../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 16 +-
.../llvm.amdgcn.image.sample.a16.dim.ll | 131 +-
.../llvm.amdgcn.image.sample.d16.dim.ll | 58 +-
.../llvm.amdgcn.image.sample.g16.encode.ll | 16 +-
.../AMDGPU/llvm.amdgcn.image.sample.g16.ll | 6 +-
.../AMDGPU/llvm.amdgcn.interp.inreg.ll | 2 +-
.../AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll | 4 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll | 7 -
.../llvm.amdgcn.raw.tbuffer.load.d16.ll | 14 -
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 115 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 115 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 115 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 115 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 115 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll | 8 -
.../llvm.amdgcn.struct.tbuffer.load.d16.ll | 16 -
.../AMDGPU/llvm.amdgcn.waitcnt.out.order.ll | 28 +-
llvm/test/CodeGen/AMDGPU/llvm.frexp.ll | 48 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll | 19 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 57 +-
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll | 497 +-
llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 22 +-
llvm/test/CodeGen/AMDGPU/lround.ll | 26 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll | 31 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll | 36 +-
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 92 +-
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 92 +-
.../CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll | 3 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 182 +-
.../AMDGPU/reassoc-mul-add-1-to-mad.ll | 40 +-
llvm/test/CodeGen/AMDGPU/repeated-divisor.ll | 14 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll | 164 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll | 59 -
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 16 +-
llvm/test/CodeGen/AMDGPU/scmp.ll | 22 +-
.../AMDGPU/select-fabs-fneg-extract.f16.ll | 54 +-
llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll | 4 +-
llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll | 19 +-
llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll | 19 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 19 +-
llvm/test/CodeGen/AMDGPU/usubsat.ll | 131 +-
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 46 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll | 44 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 44 +-
.../test/CodeGen/AMDGPU/vector-reduce-fadd.ll | 36 +-
.../test/CodeGen/AMDGPU/vector-reduce-fmul.ll | 36 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 44 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 44 +-
.../test/CodeGen/AMDGPU/vector-reduce-smax.ll | 234 +-
.../test/CodeGen/AMDGPU/vector-reduce-smin.ll | 234 +-
.../test/CodeGen/AMDGPU/vector-reduce-umax.ll | 164 +-
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 164 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 44 +-
.../test/CodeGen/AMDGPU/vector_rebroadcast.ll | 124 +-
180 files changed, 32582 insertions(+), 25197 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 3c78bb1dd7be3..75fd048a0ec3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -32,10 +32,14 @@ namespace {
/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.
static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,
- Register ValVReg, const CCValAssign &VA) {
- if (VA.getLocVT().getSizeInBits() < 32) {
+ Register ValVReg, const CCValAssign &VA,
+ bool isPhyRegSGPR, const GCNSubtarget &ST) {
+ if (VA.getLocVT().getSizeInBits() < 32 &&
+ (isPhyRegSGPR || !ST.useRealTrue16Insts())) {
// 16-bit types are reported as legal for 32-bit registers. We need to
// extend and do a 32-bit copy to avoid the verifier complaining about it.
+ //
+ // In true16 mode, we still extend to 32 if the physical reg is a sgpr
return Handler.MIRBuilder.buildAnyExt(LLT::integer(32), ValVReg).getReg(0);
}
@@ -64,14 +68,17 @@ struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
void assignValueToReg(Register ValVReg, Register PhysReg,
const CCValAssign &VA,
ISD::ArgFlagsTy Flags = {}) override {
- Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
+ const SIRegisterInfo *TRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
+ bool isPhyRegSGPR = TRI->isSGPRReg(MRI, PhysReg);
+
+ Register ExtReg = extendRegisterMin32(*this, ValVReg, VA, isPhyRegSGPR, ST);
// If this is a scalar return, insert a readfirstlane just in case the value
// ends up in a VGPR.
// FIXME: Assert this is a shader return.
- const SIRegisterInfo *TRI
- = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
- if (TRI->isSGPRReg(MRI, PhysReg)) {
+ if (isPhyRegSGPR) {
LLT Ty = MRI.getType(ExtReg);
LLT I32 = LLT::integer(32);
if (Ty != I32 && Ty != LLT::float32()) {
@@ -119,7 +126,13 @@ struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
}
void copyToReg(Register ValVReg, Register PhysReg, const CCValAssign &VA) {
- if (VA.getLocVT().getSizeInBits() < 32) {
+ const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
+ const SIRegisterInfo *TRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ bool isPhyRegSGPR = TRI->isSGPRReg(MRI, PhysReg);
+
+ if (VA.getLocVT().getSizeInBits() < 32 &&
+ (isPhyRegSGPR || !ST.useRealTrue16Insts())) {
// 16-bit types are reported as legal for 32-bit registers. We need to
// do a 32-bit copy, and truncate to avoid the verifier complaining
// about it.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 2932bbf0e7bbd..51740440c86d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -18,6 +18,10 @@ class CCIfOrigTypeShaderCCIsSGPR<CCAction A>
: CCIf<[{(!OrigTy->getScalarType()->isFloatTy() &&
!OrigTy->getScalarType()->isHalfTy()) }], A>;
+class CCIfTrue16<CCAction A> :
+ CCIf<"State.getMachineFunction().getSubtarget<GCNSubtarget>().useRealTrue16Insts()", A>;
+class CCIfNotTrue16<CCAction A> :
+ CCIf<"!State.getMachineFunction().getSubtarget<GCNSubtarget>().useRealTrue16Insts()", A>;
// Calling convention for SI
def CC_SI_Gfx : CallingConv<[
@@ -30,10 +34,18 @@ def CC_SI_Gfx : CallingConv<[
!foreach(i, !range(4, 30), !cast<Register>("SGPR"#i)) // SGPR4-29
>>>,
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>>,
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
+
CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
]>;
@@ -41,9 +53,17 @@ def RetCC_SI_Gfx : CallingConv<[
CCIfType<[i1], CCPromoteToType<i32>>,
CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
>>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
]>;
def CC_SI_SHADER : CallingConv<[
@@ -55,9 +75,17 @@ def CC_SI_SHADER : CallingConv<[
>>>,
// 32*4 + 4 is the minimum for a fetch shader consumer with 32 inputs.
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
- >>>
+ >>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-135_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
+ >>>,
]>;
@@ -68,9 +96,17 @@ def RetCC_SI_Shader : CallingConv<[
>>>,
// 32*4 + 4 is the minimum for a fetch shader with 32 outputs.
- CCIfType<[f32, f16, v2f16, bf16, v2bf16, i32, i16, v2i16] , CCAssignToReg<
+ CCIfType<[f32, v2f16, v2bf16, i32, v2i16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
- >>
+ >>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-135_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
+ >>>,
]>;
def CSR_AMDGPU_VGPRs : CalleeSavedRegs<
@@ -143,9 +179,18 @@ def CC_AMDGPU_Func : CallingConv<[
!foreach(i, !range(0, 30), !cast<Register>("SGPR"#i)) // SGPR0-29
>>>,
- CCIfType<[i32, f32, i16, f16, v2i16, v2f16, i1, bf16, v2bf16], CCAssignToReg<
+ CCIfType<[i32, f32, v2i16, v2f16, i1, v2bf16], CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
+
CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
]>;
@@ -153,9 +198,17 @@ def CC_AMDGPU_Func : CallingConv<[
def RetCC_AMDGPU_Func : CallingConv<[
CCIfType<[i1], CCPromoteToType<i32>>,
CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
- CCIfType<[i32, f32, i16, f16, v2i16, v2f16, bf16, v2bf16], CCAssignToReg<
+ CCIfType<[i32, f32, v2i16, v2f16, v2bf16], CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
]>;
def CC_AMDGPU : CallingConv<[
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 15053568da8dc..bb854fb890547 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3761,12 +3761,15 @@ SDValue SITargetLowering::LowerFormalArguments(
RC = &AMDGPU::VGPR_32RegClass;
else if (AMDGPU::SGPR_32RegClass.contains(Reg))
RC = &AMDGPU::SGPR_32RegClass;
+ else if (AMDGPU::VGPR_16RegClass.contains(Reg))
+ RC = &AMDGPU::VGPR_16RegClass;
else
llvm_unreachable("Unexpected register class in LowerFormalArguments!");
Reg = MF.addLiveIn(Reg, RC);
SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
- if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
+ if (Arg.Flags.isInReg() &&
+ (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::VGPR_16RegClass)) {
// FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
// they will read physical regs before any side effect instructions.
SDValue ReadFirstLane =
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
index ddc1f68e8fbe0..4eb06b667b238 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
@@ -33,11 +33,23 @@ define i16 @v_and_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_and_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_and_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_and_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_and_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_and_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_and_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_and_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = and i16 %num, %den
ret i16 %result
@@ -1134,6 +1146,3 @@ define <2 x i256> @v_and_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %and
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 5a8027bf75881..c1767b5598a85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -38,9 +38,9 @@ define i8 @v_ashr_i8(i8 %value, i8 %amount) {
; GFX11-TRUE16-LABEL: v_ashr_i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_ashr_i8:
@@ -2055,18 +2055,18 @@ define <4 x i2> @v_ashr_v4i2(<4 x i2> %value, <4 x i2> %amount) {
; GFX11-TRUE16-LABEL: v_ashr_v4i2:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 2
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v5.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v7.l, 3
-; GFX11-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 2
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v2.l, v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v3.l, v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v5.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v6.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v7.l, 3
+; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v0, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v1, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v6, v2, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v7, v3, 0, 2
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v2.l, v2.h, v6.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v3.l, v3.h, v7.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_ashr_v4i2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
index ce4de6e628573..641f2ef840cc3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
@@ -159,10 +159,10 @@ define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
; GFX11-CONTRACT-TRUE16-LABEL: test_half_add_mul:
; GFX11-CONTRACT-TRUE16: ; %bb.0: ; %.entry
; GFX11-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v2.l, v2.l, v3.l, v4.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v0.h, v2.l, v3.l, v4.l
; GFX11-CONTRACT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v1.l
-; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v0.h, v0.l, v1.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-CONTRACT-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-CONTRACT-FAKE16-LABEL: test_half_add_mul:
@@ -236,10 +236,10 @@ define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e)
; GFX11-CONTRACT-TRUE16-LABEL: test_half_add_mul_rhs:
; GFX11-CONTRACT-TRUE16: ; %bb.0: ; %.entry
; GFX11-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v2.l, v2.l, v3.l, v4.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v0.h, v2.l, v3.l, v4.l
; GFX11-CONTRACT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v1.l
-; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v0.h, v0.l, v1.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-CONTRACT-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-CONTRACT-FAKE16-LABEL: test_half_add_mul_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index c16a8d89f310c..52bda12f528dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -81,7 +81,7 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX12-NEXT: s_cmp_u_f32 s0, 0
; GFX12-NEXT: s_cselect_b32 s0, s2, s1
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fadd_bf16_ss:
@@ -100,7 +100,7 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
index 808bb42378841..dcc19682b9ec8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fadd_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fadd_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_add_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fadd_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_add_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fadd_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_add_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%fadd = fadd half %a, %b
ret half %fadd
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 1efddd0f7d383..493aeb9689f31 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -82,7 +82,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fcanonicalize_bf16_s:
@@ -101,7 +101,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
index a25a30fa5a8a3..be230801dd17b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
@@ -62,7 +62,7 @@ define half @fceil_s16_s(half inreg %val) {
; GFX12-NEXT: s_ceil_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.ceil.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
index c1785c0f62b12..8bff476594d26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
@@ -62,7 +62,7 @@ define half @ffloor_s16_s(half inreg %val) {
; GFX12-NEXT: s_floor_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.floor.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index cbca4c98f58e9..75ce7cdd965fd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -87,7 +87,7 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX12-NEXT: s_cmp_u_f32 s2, 0
; GFX12-NEXT: s_cselect_b32 s0, s1, s0
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fma_bf16_sss:
@@ -107,7 +107,7 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX1250-NEXT: s_cmp_u_f32 s2, 0
; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index e6a93c653cedf..672f710078e09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -1539,7 +1539,7 @@ define amdgpu_ps half @fma_s16_uniform(half inreg %a, half inreg %b, half inreg
; GFX12-LABEL: fma_s16_uniform:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-NEXT: ; return to shader part epilog
%fma = call half @llvm.fma.f16(half %a, half %b, half %c)
ret half %fma
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
index 81ddff1dd2a4f..7fabdbb717e0f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
@@ -38,7 +38,7 @@ define amdgpu_ps half @fmed3_s16_uniform_salu_use(half inreg %a, half inreg %b,
; GFX12-NEXT: s_add_f16 s0, s0, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
%result = call half @llvm.amdgcn.fmed3.f16(half %a, half %b, half %c)
%add = fadd half %result, %d
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 493374e422da1..7a337cb7b1dca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -81,7 +81,7 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX12-NEXT: s_cmp_u_f32 s0, 0
; GFX12-NEXT: s_cselect_b32 s0, s2, s1
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fmul_bf16_ss:
@@ -100,7 +100,7 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
index a470767e80a39..8fc59ce727916 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fmul_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fmul_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mul_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fmul_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fmul_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%result = fmul half %a, %b
ret half %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index a66441bcebc18..6b4bee8124a36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -7,51 +7,97 @@
; RUN: llc -global-isel -mtriple=amdgpu12.50-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
define amdgpu_ps bfloat @fptrunc_f32_to_bf16_s(float inreg %a) {
-; GFX11-LABEL: fptrunc_f32_to_bf16_s:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s2, s0, 0
-; GFX11-NEXT: s_add_i32 s1, s1, s0
-; GFX11-NEXT: s_bitset1_b32 s0, 22
-; GFX11-NEXT: s_addk_i32 s1, 0x7fff
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-FAKE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f32_to_bf16_s:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX12-NEXT: s_add_co_i32 s1, s1, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX12-NEXT: s_cmp_u_f32 s0, 0
-; GFX12-NEXT: s_cselect_b32 s0, s2, s1
-; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-TRUE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f32_to_bf16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc float %a to bfloat
ret bfloat %result
}
@@ -101,110 +147,215 @@ define amdgpu_ps bfloat @fptrunc_f32_to_bf16_v(float %a) {
}
define amdgpu_ps bfloat @fptrunc_f64_to_bf16_s(double inreg %a) {
-; GFX11-LABEL: fptrunc_f64_to_bf16_s:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX11-NEXT: v_readfirstlane_b32 s1, v2
-; GFX11-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX11-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s2, s2, s1
-; GFX11-NEXT: s_cmp_lg_u32 s0, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_and_b32 s2, s2, 1
-; GFX11-NEXT: s_or_b32 s0, s0, 1
-; GFX11-NEXT: s_add_i32 s0, s1, s0
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s2, s0, 0
-; GFX11-NEXT: s_add_i32 s1, s1, s0
-; GFX11-NEXT: s_bitset1_b32 s0, 22
-; GFX11-NEXT: s_addk_i32 s1, 0x7fff
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX11-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-FAKE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f64_to_bf16_s:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX12-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX12-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX12-NEXT: v_readfirstlane_b32 s1, v2
-; GFX12-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX12-NEXT: s_cselect_b32 s2, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s2, s2, s1
-; GFX12-NEXT: s_cmp_lg_u32 s0, 0
-; GFX12-NEXT: s_cselect_b32 s0, -1, 0
-; GFX12-NEXT: s_and_b32 s2, s2, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, s0, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s0, s1, s0
-; GFX12-NEXT: s_cmp_lg_u32 s2, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cselect_b32 s0, s1, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s1, s1, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX12-NEXT: s_cmp_u_f32 s0, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cselect_b32 s0, s2, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX11-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-TRUE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f64_to_bf16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX1250-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1250-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s2, s2, s1
-; GFX1250-NEXT: s_cmp_lg_u32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, 1
-; GFX1250-NEXT: s_or_b32 s0, s0, 1
-; GFX1250-NEXT: s_add_co_i32 s0, s1, s0
-; GFX1250-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX12-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX12-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX1250-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX1250-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc double %a to bfloat
ret bfloat %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index d34b222eb56ea..8b251d1beb541 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -17,24 +17,43 @@ define amdgpu_ps half @fptrunc_f32_to_f16_uniform(float inreg %a) {
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f32_to_f16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f32_to_f16_uniform:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_cvt_f16_f32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-TRUE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc float %a to half
ret half %result
}
@@ -130,165 +149,325 @@ define amdgpu_ps float @fptrunc_f64_to_f32_div(double %a) {
}
define amdgpu_ps half @fptrunc_f64_to_f16_uniform(double inreg %a) {
-; GFX11-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX11-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX11-NEXT: s_addk_i32 s2, 0xfc10
-; GFX11-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX11-NEXT: s_or_b32 s0, s4, s0
-; GFX11-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s3, s0
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, 1, s2
-; GFX11-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX11-NEXT: s_max_i32 s4, s4, 0
-; GFX11-NEXT: s_lshl_b32 s3, s3, 9
-; GFX11-NEXT: s_min_i32 s4, s4, 13
-; GFX11-NEXT: s_lshl_b32 s7, s2, 12
-; GFX11-NEXT: s_lshr_b32 s6, s5, s4
-; GFX11-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX11-NEXT: s_lshl_b32 s4, s6, s4
-; GFX11-NEXT: s_or_b32 s0, s0, s7
-; GFX11-NEXT: s_cmp_lg_u32 s4, s5
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s4, s6, s4
-; GFX11-NEXT: s_cmp_lt_i32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s0, s4, s0
-; GFX11-NEXT: s_and_b32 s4, s0, 7
-; GFX11-NEXT: s_lshr_b32 s0, s0, 2
-; GFX11-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_gt_i32 s4, 5
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s4, s5, s4
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s4
-; GFX11-NEXT: s_cmp_gt_i32 s2, 30
-; GFX11-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX11-NEXT: s_cmpk_eq_i32 s2, 0x40f
-; GFX11-NEXT: s_cselect_b32 s0, s3, s0
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX11-FAKE16-NEXT: s_addk_i32 s2, 0xfc10
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, 1, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX11-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX11-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX11-FAKE16-NEXT: s_cmpk_eq_i32 s2, 0x40f
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX12-NEXT: s_lshr_b32 s3, s1, 8
-; GFX12-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX12-NEXT: s_addk_co_i32 s2, 0xfc10
-; GFX12-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX12-NEXT: s_or_b32 s0, s4, s0
-; GFX12-NEXT: s_cselect_b32 s0, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s3, s0
-; GFX12-NEXT: s_cselect_b32 s3, 1, 0
-; GFX12-NEXT: s_sub_co_i32 s4, 1, s2
-; GFX12-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX12-NEXT: s_max_i32 s4, s4, 0
-; GFX12-NEXT: s_lshl_b32 s3, s3, 9
-; GFX12-NEXT: s_min_i32 s4, s4, 13
-; GFX12-NEXT: s_lshl_b32 s7, s2, 12
-; GFX12-NEXT: s_lshr_b32 s6, s5, s4
-; GFX12-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX12-NEXT: s_lshl_b32 s4, s6, s4
-; GFX12-NEXT: s_or_b32 s0, s0, s7
-; GFX12-NEXT: s_cmp_lg_u32 s4, s5
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s4, s6, s4
-; GFX12-NEXT: s_cmp_lt_i32 s2, 1
-; GFX12-NEXT: s_cselect_b32 s0, s4, s0
-; GFX12-NEXT: s_and_b32 s4, s0, 7
-; GFX12-NEXT: s_lshr_b32 s0, s0, 2
-; GFX12-NEXT: s_cmp_eq_u32 s4, 3
-; GFX12-NEXT: s_cselect_b32 s5, 1, 0
-; GFX12-NEXT: s_cmp_gt_i32 s4, 5
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s4, s5, s4
-; GFX12-NEXT: s_cmp_lg_u32 s4, 0
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_add_co_i32 s0, s0, s4
-; GFX12-NEXT: s_cmp_gt_i32 s2, 30
-; GFX12-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX12-NEXT: s_cmp_eq_u32 s2, 0x40f
-; GFX12-NEXT: s_cselect_b32 s0, s3, s0
-; GFX12-NEXT: s_lshr_b32 s1, s1, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX11-TRUE16-NEXT: s_addk_i32 s2, 0xfc10
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, 1, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX11-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX11-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX11-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s2, 0x40f
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX1250-NEXT: s_lshr_b32 s3, s1, 8
-; GFX1250-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX1250-NEXT: s_addk_co_i32 s2, 0xfc10
-; GFX1250-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX1250-NEXT: s_or_b32 s0, s4, s0
-; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s0, s3, s0
-; GFX1250-NEXT: s_cselect_b32 s3, 1, 0
-; GFX1250-NEXT: s_sub_co_i32 s4, 1, s2
-; GFX1250-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX1250-NEXT: s_max_i32 s4, s4, 0
-; GFX1250-NEXT: s_lshl_b32 s3, s3, 9
-; GFX1250-NEXT: s_min_i32 s4, s4, 13
-; GFX1250-NEXT: s_lshl_b32 s7, s2, 12
-; GFX1250-NEXT: s_lshr_b32 s6, s5, s4
-; GFX1250-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX1250-NEXT: s_lshl_b32 s4, s6, s4
-; GFX1250-NEXT: s_or_b32 s0, s0, s7
-; GFX1250-NEXT: s_cmp_lg_u32 s4, s5
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s4, s6, s4
-; GFX1250-NEXT: s_cmp_lt_i32 s2, 1
-; GFX1250-NEXT: s_cselect_b32 s0, s4, s0
-; GFX1250-NEXT: s_and_b32 s4, s0, 7
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 2
-; GFX1250-NEXT: s_cmp_eq_u32 s4, 3
-; GFX1250-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-NEXT: s_cmp_gt_i32 s4, 5
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_add_co_i32 s0, s0, s4
-; GFX1250-NEXT: s_cmp_gt_i32 s2, 30
-; GFX1250-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX1250-NEXT: s_cmp_eq_u32 s2, 0x40f
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s0
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX1250-NEXT: s_or_b32 s0, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX12-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX12-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX12-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX12-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX12-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX12-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX12-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX12-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX12-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX12-FAKE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX12-TRUE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX12-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX12-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX12-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX12-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX12-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX12-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX12-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX12-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX12-TRUE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX1250-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-FAKE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX1250-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX1250-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX1250-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX1250-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX1250-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX1250-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX1250-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-TRUE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX1250-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX1250-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX1250-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX1250-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX1250-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX1250-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX1250-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX1250-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX1250-TRUE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc double %a to half
ret half %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index c49e892ee6c47..6f3407b9c02e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -987,31 +987,29 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX11-TRUE16-LABEL: v_fshl_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v2.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v4.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v4.l, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 9, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v0.h, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index b5aa360bb8d14..69a29d5c3c5fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -985,32 +985,32 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX11-TRUE16-LABEL: v_fshr_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v2.l, -1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v3.l, -1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v3.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v0.h, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3864,10 +3864,9 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
index c26b2de6938fa..435730cec9626 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fsub_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_sub_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fsub_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_sub_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fsub_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_sub_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fsub_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_sub_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%fsub = fsub half %a, %b
ret half %fsub
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 0f963ba85b73e..445fb8061c3a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -255,9 +255,9 @@ define amdgpu_ps void @insertelement_s_v2i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: s_and_b32 s1, s4, 1
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v0.l
; GFX11-NEXT: s_lshl_b32 s1, s1, 4
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_not1_b32 s0, s0, s2
@@ -431,7 +431,7 @@ define amdgpu_ps void @insertelement_s_v2i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
@@ -612,11 +612,11 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v0
; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
@@ -702,7 +702,7 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v1, v2.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
@@ -982,7 +982,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_and_b32 s2, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s2, s2, 4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_lshl_b32 s3, 0xffff, s2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1233,7 +1233,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v2, 1, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v3, v0.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1248,8 +1248,8 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
@@ -1466,7 +1466,7 @@ define amdgpu_ps void @insertelement_v_v4i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
@@ -1585,21 +1585,20 @@ define amdgpu_ps void @insertelement_v_v4i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: v_and_b32_e32 v4, 1, v3
; GFX11-NEXT: v_lshrrev_b32_e32 v6, 1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v4
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v3, v5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
@@ -1943,15 +1942,14 @@ define amdgpu_ps void @insertelement_s_v8i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_and_b32 s5, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s4, s5, 4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_movrels_b32 s6, s0
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_and_not1_b32 s5, s6, s5
-; GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: v_lshl_or_b32 v6, v0, s4, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2265,29 +2263,29 @@ define amdgpu_ps void @insertelement_s_v8i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v8i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
; GFX11-NEXT: v_and_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 4, v5
+; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_lshlrev_b32 v5, 4, v5
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_not_b32_e32 v5, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
@@ -2540,7 +2538,7 @@ define amdgpu_ps void @insertelement_v_v8i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v8i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
@@ -2682,12 +2680,12 @@ define amdgpu_ps void @insertelement_v_v8i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-NEXT: v_mov_b32_e32 v9, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3129,20 +3127,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_s(ptr addrspace(4) inreg %ptr, i
; GFX11-NEXT: s_and_b32 s0, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v8, v0.l
; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s0
-; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v9, 0
-; GFX11-NEXT: v_mov_b32_e32 v11, 0
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v11, 0
+; GFX11-NEXT: v_mov_b32_e32 v9, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_movrels_b32 s2, s8
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX11-NEXT: s_and_not1_b32 s1, s2, s1
-; GFX11-NEXT: v_mov_b32_e32 v1, s9
+; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX11-NEXT: v_lshl_or_b32 v12, v8, s0, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
-; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
-; GFX11-NEXT: v_mov_b32_e32 v6, s14
+; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_movreld_b32_e32 v0, v12
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
@@ -3627,22 +3625,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[0:7], s[2:3], 0x0
; GFX11-NEXT: v_lshrrev_b32_e32 v14, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_mov_b32 v13, 0
; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_mov_b32_e32 v13, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_not_b32_e32 v1, v11
-; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
-; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
-; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_lshlrev_b32 v0, v1, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX11-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v5, s3
+; GFX11-NEXT: v_mov_b32_e32 v4, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
@@ -3653,19 +3649,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v1, v11
+; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v15, v10, v1, v0
; GFX11-NEXT: v_mov_b32_e32 v10, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_cndmask_b32 v1, v3, v15
; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v15, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
@@ -4031,14 +4028,14 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
; GFX11-NEXT: global_load_b128 v[7:10], v[0:1], off offset:16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v11, 16 :: v_dual_lshlrev_b32 v0, s0, v0
-; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX11-NEXT: v_mov_b32_e32 v12, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: v_mov_b32_e32 v11, 16
; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_movrels_b32_e32 v1, v3
@@ -4248,36 +4245,36 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v3
; GFX11-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX11-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v14, 16
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 16
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_lshlrev_b32 v2, v3, v2
-; GFX11-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
+; GFX11-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_lshlrev_b32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v3, v12
; GFX11-NEXT: s_waitcnt vmcnt(1)
; GFX11-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_cndmask_b32 v1, v4, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v10, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
; GFX11-NEXT: v_and_or_b32 v16, v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, v16, s6
; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v16, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
index ddf02e41e6ce8..c4d316ca1a71c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
@@ -62,7 +62,7 @@ define half @intrinsic_trunc_s16_s(half inreg %val) {
; GFX12-NEXT: s_trunc_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.trunc.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
index 1e3cbb574470b..3d83f6d4413be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11NSA %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
@@ -51,9 +51,31 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_swap_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_swap_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -65,10 +87,9 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -126,9 +147,31 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -140,10 +183,9 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -201,9 +243,31 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_sub_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_sub_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -215,10 +279,9 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -276,9 +339,31 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_smin_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_smin_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -290,10 +375,9 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -352,9 +436,31 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_umin_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_umin_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -366,10 +472,9 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -427,9 +532,31 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_smax_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_smax_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -441,10 +568,9 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -502,9 +628,31 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_umax_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_umax_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -516,10 +664,9 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -577,9 +724,31 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_and_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_and_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -591,10 +760,9 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -652,9 +820,31 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_or_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_or_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -666,10 +856,9 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -727,9 +916,31 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_xor_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_xor_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -741,10 +952,9 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -802,9 +1012,31 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_inc_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_inc_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -816,10 +1048,9 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -877,9 +1108,31 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_dec_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_dec_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -891,10 +1144,9 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -956,9 +1208,33 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -971,11 +1247,10 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1035,9 +1310,31 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1049,11 +1346,9 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1122,9 +1417,34 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_3d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1136,15 +1456,12 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1213,9 +1530,34 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_cube
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_cube
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1227,15 +1569,12 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1295,9 +1634,31 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1darray
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1309,11 +1670,9 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1382,9 +1741,34 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2darray
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1396,15 +1780,12 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1473,9 +1854,34 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2dmsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2dmsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1487,15 +1893,12 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1566,9 +1969,34 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2darraymsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[COPY12]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2darraymsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1580,16 +2008,12 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[COPY12]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1647,9 +2071,31 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1d_slc
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1d_slc
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1661,10 +2107,9 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1728,9 +2173,33 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_2d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1743,12 +2212,10 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1821,9 +2288,36 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_3d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1836,16 +2330,13 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1920,9 +2411,36 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_2darraymsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[COPY13:%[0-9]+]]:_(i16) = COPY $vgpr5_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[COPY13]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_2darraymsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1935,17 +2453,13 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[COPY13]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
index 61006f8c1eab6..b55233b3cfc30 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
@@ -67,7 +67,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -83,11 +83,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -98,7 +96,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -114,11 +112,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -196,7 +192,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -212,13 +208,9 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -229,7 +221,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -245,13 +237,9 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -338,7 +326,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -354,18 +342,12 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -376,7 +358,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -392,18 +374,12 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -490,7 +466,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_cube
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -506,18 +482,12 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -528,7 +498,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_cube
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -544,18 +514,12 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -633,7 +597,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_1darray
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -649,13 +613,9 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -666,7 +626,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_1darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -682,13 +642,9 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -775,7 +731,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_2darray
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -791,18 +747,12 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -813,7 +763,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_2darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -829,18 +779,12 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -921,7 +865,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_c_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -938,13 +882,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -954,7 +896,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_c_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -971,13 +913,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1059,7 +999,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_c_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1076,15 +1016,11 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1094,7 +1030,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_c_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1111,15 +1047,11 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1196,7 +1128,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1212,13 +1144,9 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1229,7 +1157,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1245,13 +1173,9 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1338,7 +1262,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1354,18 +1278,12 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1376,7 +1294,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1392,18 +1310,12 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1486,7 +1398,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1503,15 +1415,11 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1521,7 +1429,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1538,15 +1446,11 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1636,7 +1540,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1653,20 +1557,14 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1676,7 +1574,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1693,20 +1591,14 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1788,7 +1680,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_b_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1804,15 +1696,11 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1823,7 +1711,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_b_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1839,15 +1727,11 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1934,7 +1818,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_b_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1950,18 +1834,12 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1972,7 +1850,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_b_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1988,18 +1866,12 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2086,7 +1958,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_b_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2102,18 +1974,14 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2123,7 +1991,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_b_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2139,18 +2007,14 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2240,7 +2104,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_b_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2256,21 +2120,15 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2280,7 +2138,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_b_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2296,21 +2154,15 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2396,7 +2248,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_b_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2412,18 +2264,12 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2434,7 +2280,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_b_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2450,18 +2296,12 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2554,7 +2394,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_b_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2570,22 +2410,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2596,7 +2428,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_b_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2612,22 +2444,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2718,7 +2542,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_b_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2734,21 +2558,15 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2758,7 +2576,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_b_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2774,21 +2592,15 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2884,7 +2696,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_b_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2900,25 +2712,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2928,7 +2732,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_b_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2944,25 +2748,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3050,7 +2846,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3066,19 +2862,13 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3089,7 +2879,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3105,19 +2895,13 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3216,7 +3000,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3232,27 +3016,15 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3263,7 +3035,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3279,27 +3051,15 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3419,7 +3179,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3435,40 +3195,22 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3480,7 +3222,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3496,40 +3238,22 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3623,7 +3347,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3640,21 +3364,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3664,7 +3382,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3681,21 +3399,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3797,7 +3509,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3814,29 +3526,17 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3846,7 +3546,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3863,29 +3563,17 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3977,7 +3665,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3993,22 +3681,14 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4019,7 +3699,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4035,22 +3715,14 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4157,7 +3829,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4173,32 +3845,18 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4209,7 +3867,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4225,32 +3883,18 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4347,7 +3991,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4364,24 +4008,16 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4391,7 +4027,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4408,24 +4044,16 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4535,7 +4163,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4552,34 +4180,20 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4589,7 +4203,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4606,35 +4220,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4722,7 +4322,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4738,19 +4338,13 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4761,7 +4355,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4777,19 +4371,13 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4888,7 +4476,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4904,27 +4492,15 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4935,7 +4511,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4951,27 +4527,15 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5064,7 +4628,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5081,21 +4645,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5105,7 +4663,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5122,21 +4680,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5238,7 +4790,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5255,29 +4807,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5287,7 +4827,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5304,29 +4844,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5418,7 +4946,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5434,22 +4962,14 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5460,7 +4980,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5476,22 +4996,14 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5598,7 +5110,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5614,32 +5126,18 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5650,7 +5148,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5666,32 +5164,18 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5788,7 +5272,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5805,24 +5289,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5832,7 +5308,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5849,24 +5325,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5976,7 +5444,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5993,34 +5461,20 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6030,7 +5484,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6047,35 +5501,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6152,7 +5592,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_l_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6168,13 +5608,9 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6185,7 +5621,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_l_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6201,13 +5637,9 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6294,7 +5726,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_l_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6310,18 +5742,12 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6332,7 +5758,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_l_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6348,18 +5774,12 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6442,7 +5862,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_l_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6459,15 +5879,11 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6477,7 +5893,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_l_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6494,15 +5910,11 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6592,7 +6004,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_l_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6609,20 +6021,14 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6632,7 +6038,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_l_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6649,20 +6055,14 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6737,7 +6137,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_lz_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6753,11 +6153,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6768,7 +6166,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_lz_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6784,11 +6182,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6866,7 +6262,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_lz_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6882,13 +6278,9 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6899,7 +6291,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_lz_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6915,13 +6307,9 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7002,7 +6390,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_lz_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7019,13 +6407,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7035,7 +6421,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_lz_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7052,13 +6438,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7140,7 +6524,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_lz_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7157,15 +6541,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7175,7 +6555,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_lz_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7192,15 +6572,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7307,7 +6683,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V1
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7325,42 +6701,28 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX11-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V1
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7378,36 +6740,22 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
main_body:
@@ -7514,7 +6862,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V2
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7532,36 +6880,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7569,7 +6903,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V2
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7587,36 +6921,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
index 4934f93ac18df..6ee08839fbb93 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
@@ -42,7 +42,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_1d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -58,19 +58,13 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -81,7 +75,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_1d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -97,19 +91,13 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -166,7 +154,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_2d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -182,27 +170,15 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -213,7 +189,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_2d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -229,27 +205,15 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -317,7 +281,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_3d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -333,40 +297,22 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -378,7 +324,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_3d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -394,40 +340,22 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
index 2ab7e0d348f41..12df216b7d7e2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
@@ -41,7 +41,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -57,18 +57,14 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -78,7 +74,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -94,18 +90,14 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -160,7 +152,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -176,25 +168,17 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -204,7 +188,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -220,25 +204,17 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -303,7 +279,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr6, $vgpr7, $vgpr8, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -319,36 +295,24 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -359,7 +323,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr6, $vgpr7, $vgpr8, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -375,36 +339,24 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -457,7 +409,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -474,19 +426,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -496,7 +444,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -513,19 +461,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -582,7 +526,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -599,26 +543,18 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -628,7 +564,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -645,27 +581,19 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -717,7 +645,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -733,20 +661,16 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -756,7 +680,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -772,20 +696,16 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -842,7 +762,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -858,27 +778,19 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -888,7 +800,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -904,28 +816,20 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -979,7 +883,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -996,21 +900,17 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1020,7 +920,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1037,22 +937,18 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1112,7 +1008,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1129,29 +1025,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1161,7 +1049,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1178,29 +1066,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1250,7 +1130,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1266,18 +1146,14 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1287,7 +1163,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1303,18 +1179,14 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1369,7 +1241,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1385,25 +1257,17 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1413,7 +1277,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1429,25 +1293,17 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1499,7 +1355,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1516,19 +1372,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1538,7 +1390,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1555,19 +1407,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1624,7 +1472,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1641,26 +1489,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1670,7 +1510,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1687,27 +1527,19 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1759,7 +1591,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1775,20 +1607,16 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1798,7 +1626,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1814,20 +1642,16 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1884,7 +1708,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1900,27 +1724,19 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1930,7 +1746,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1946,28 +1762,20 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2021,7 +1829,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2038,21 +1846,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2062,7 +1866,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2079,22 +1883,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2154,7 +1954,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2171,29 +1971,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2203,7 +1995,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2220,29 +2012,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2300,7 +2084,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V1
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2318,36 +2102,28 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX11-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V1
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2365,30 +2141,22 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
main_body:
@@ -2444,7 +2212,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V2
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2462,30 +2230,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2493,7 +2253,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V2
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2511,30 +2271,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index 9657f2ec27268..5cc6d9c6b9f7b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -72,7 +72,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
;
; GFX12-LABEL: name: image_store_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -85,10 +85,8 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
index ac1b50a967ebe..177edf7dae535 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
@@ -13,7 +13,7 @@ define amdgpu_ps half @s_fract_f16(half inreg %src) {
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: s_add_f16 s0, s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
%fract = call half @llvm.amdgcn.fract.f16(half %src)
%res = fadd half %fract, %fract
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index 394d145f6064d..33513214e779b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -1420,6 +1420,7 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX11-TRUE16-LABEL: atomic_add_i32_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1428,7 +1429,7 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1528,6 +1529,7 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX11-TRUE16-LABEL: atomic_add_i32_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1536,7 +1538,7 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1736,6 +1738,7 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-LABEL: atomic_add_i32_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1744,7 +1747,7 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1844,6 +1847,7 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-LABEL: atomic_add_i32_2dmsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1852,7 +1856,7 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1951,9 +1955,9 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
;
; GFX11-TRUE16-LABEL: atomic_add_i32_2darraymsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1962,7 +1966,7 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, v[5:6], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3535,6 +3539,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX11-TRUE16-LABEL: atomic_add_i64_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3543,7 +3548,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3643,6 +3648,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX11-TRUE16-LABEL: atomic_add_i64_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3651,7 +3657,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3851,6 +3857,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX11-TRUE16-LABEL: atomic_add_i64_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3859,7 +3866,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3959,6 +3966,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX11-TRUE16-LABEL: atomic_add_i64_2dmsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3967,7 +3975,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -4066,9 +4074,9 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
;
; GFX11-TRUE16-LABEL: atomic_add_i64_2darraymsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -4077,7 +4085,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[6:7], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index 0cd3a5d235ea6..32eb64298cd40 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -55,9 +55,11 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: gather4_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -69,8 +71,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -100,9 +100,11 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-TRUE16-LABEL: gather4_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -114,8 +116,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -200,9 +200,12 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -214,9 +217,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -247,9 +248,11 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_cube:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -261,8 +264,6 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -349,9 +350,12 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -363,9 +367,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -396,9 +398,11 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_2darray:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -410,8 +414,6 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -494,9 +496,11 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_c_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -508,8 +512,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -539,9 +541,11 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_c_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -553,8 +557,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -639,9 +641,12 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-TRUE16-LABEL: gather4_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -653,9 +658,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -686,9 +689,11 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-TRUE16-LABEL: gather4_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -700,8 +705,6 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -788,9 +791,11 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_c_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -802,8 +807,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -835,9 +838,11 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_c_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -849,8 +854,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -937,9 +940,11 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_b_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -951,8 +956,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -984,9 +987,11 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_b_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -998,8 +1003,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_b v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1086,9 +1089,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-TRUE16-LABEL: gather4_c_b_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1100,8 +1105,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1133,9 +1136,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-TRUE16-LABEL: gather4_c_b_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1147,8 +1152,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1237,9 +1240,12 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_b_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1251,9 +1257,7 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1286,9 +1290,11 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_b_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1300,8 +1306,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1392,9 +1396,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-TRUE16-LABEL: gather4_c_b_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1406,8 +1412,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1441,9 +1445,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: gather4_c_b_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1455,8 +1461,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1540,6 +1544,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1552,7 +1557,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 7ca3ae1561b2f..b633a37dd22be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -45,10 +45,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
;
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -57,7 +56,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_load v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-TRUE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -179,9 +178,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -237,9 +236,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -361,9 +360,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -419,9 +418,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index cd1b7227e427a..b337e5db23f5d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -46,6 +46,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -54,7 +55,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_load v[0:3], [v0, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX11-TRUE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -175,10 +176,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
@@ -228,10 +229,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
@@ -343,10 +344,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
@@ -396,10 +397,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 096cb9ff625da..d8866497d2169 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -496,11 +496,12 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
-; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[4:7]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
@@ -554,11 +555,12 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
-; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[4:7]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
index 225ae801fc12b..42469f8f78c47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
@@ -397,7 +397,7 @@ define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #
; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-TRUE16-NEXT: s_add_f16 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: v_interp_f16_imm_params:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 5739d01e61d95..74173ad39f198 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -700,19 +700,21 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-LABEL: name: general_case_load_with_waterfall
; CHECK45: bb.1 (%ir-block.0):
; CHECK45-NEXT: successors: %bb.2(0x80000000)
- ; CHECK45-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; CHECK45-NEXT: liveins: $vgpr0, $vgpr1, $vgpr3, $vgpr4, $vgpr5, $vgpr2_lo16
; CHECK45-NEXT: {{ $}}
; CHECK45-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
- ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY $vgpr2_lo16
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; CHECK45-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY2]], %subreg.lo16, [[DEF]], %subreg.hi16
; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8191
; CHECK45-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; CHECK45-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY4]], [[COPY6]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[V_AND_B32_e64_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[V_AND_B32_e64_]], %subreg.sub1
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_2]]
@@ -721,17 +723,17 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY3]], [[COPY7]], [[COPY9]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_3]]
- ; CHECK45-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 [[COPY10]], [[REG_SEQUENCE1]], implicit $exec
+ ; CHECK45-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 [[COPY10]], [[REG_SEQUENCE2]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_4]]
- ; CHECK45-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[COPY2]], implicit $exec
+ ; CHECK45-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[REG_SEQUENCE1]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_5]]
; CHECK45-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[COPY5]], implicit $exec
; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
; CHECK45-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub1
; CHECK45-NEXT: [[V_OR3_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR3_B32_e64 [[COPY14]], [[V_LSHLREV_B32_e64_]], [[V_LSHLREV_B32_e64_1]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[V_LSHL_OR_B32_e64_]], %subreg.sub1, [[COPY13]], %subreg.sub2, [[V_OR3_B32_e64_]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[V_LSHL_OR_B32_e64_]], %subreg.sub1, [[COPY13]], %subreg.sub2, [[V_OR3_B32_e64_]], %subreg.sub3
; CHECK45-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; CHECK45-NEXT: {{ $}}
@@ -742,11 +744,11 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_LSHL_OR_B32_e64_]], implicit $exec
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_OR3_B32_e64_]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
- ; CHECK45-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
- ; CHECK45-NEXT: [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
- ; CHECK45-NEXT: [[COPY19:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]].sub0_sub1
+ ; CHECK45-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]].sub2_sub3
+ ; CHECK45-NEXT: [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE4]].sub0_sub1
+ ; CHECK45-NEXT: [[COPY19:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE4]].sub2_sub3
; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY19]], [[COPY17]], implicit $exec
; CHECK45-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -755,7 +757,7 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: bb.3:
; CHECK45-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; CHECK45-NEXT: {{ $}}
- ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY15]], [[REG_SEQUENCE3]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY15]], [[REG_SEQUENCE4]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
; CHECK45-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; CHECK45-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; CHECK45-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
index 28d1241adb67a..525d43ecbd6e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
@@ -47,8 +47,9 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -347,8 +348,9 @@ define amdgpu_ps half @raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr
; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.5:
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY11]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -476,8 +478,10 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__sgpr_voffset__sgpr
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
index 63bce3c9706b2..3e7c1fedea977 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
@@ -808,8 +808,9 @@ define amdgpu_ps half @raw_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -1149,8 +1150,9 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX1200-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.5:
- ; GFX1200-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1200-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; GFX1250-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX1250: bb.1 (%ir-block.0):
@@ -1196,8 +1198,9 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.5:
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index e355992416d1f..5d850b07311b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -36,19 +36,18 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -85,18 +84,17 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index a9ada36d4b2ff..1be7dd9dca0bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -960,19 +960,18 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
;
; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 02179b115562a..7ea242c7f66e0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -684,8 +684,9 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -1007,8 +1008,9 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.5:
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
index 358a570c1dc15..ec6e714754c70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
@@ -46,8 +46,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
@@ -345,8 +346,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.5:
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY11]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
@@ -394,8 +396,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret half %val
}
@@ -443,8 +446,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret half %val
}
@@ -492,8 +496,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret half %val
}
@@ -541,8 +546,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret half %val
}
@@ -593,8 +599,10 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__sgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 4225ab16bac30..8ff542edd3252 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -36,19 +36,18 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -324,41 +323,40 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -478,44 +476,43 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -637,45 +634,44 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY9]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -723,19 +719,18 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@@ -774,19 +769,18 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@@ -825,19 +819,18 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@@ -876,19 +869,18 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
index ce4599e7e1e05..7fd96fc64af5f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
@@ -36,9 +36,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
;
; GFX12-LABEL: name: raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -46,7 +46,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
ret void
@@ -151,9 +153,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -161,6 +163,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -170,11 +174,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -183,7 +187,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -303,9 +307,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -313,6 +317,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -322,11 +328,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -338,7 +344,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -460,9 +466,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -470,6 +476,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
@@ -480,11 +488,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -496,7 +504,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index 1588f254b801d..cf71b26c5bad4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -52,8 +52,9 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.struct.buffer.load.format.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -455,8 +456,9 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%voffset = add i32 %voffset.base, 4095
%val = call half @llvm.amdgcn.struct.buffer.load.format.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
@@ -512,9 +514,8 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; GFX12-NEXT: $vgpr0 = COPY [[COPY8]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call i16 @llvm.amdgcn.struct.buffer.load.format.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
%fval = bitcast i16 %val to half
ret half %fval
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
index 8202e72aab96b..86e0b2fe9bb16 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
@@ -996,8 +996,9 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
- ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]].lo16
+ ; GFX1200-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; GFX1250-LABEL: name: struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX1250: bb.1 (%ir-block.0):
@@ -1013,8 +1014,9 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 8473cbb2ddceb..5c9bec56225be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -40,21 +40,20 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr2, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY9]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -455,9 +454,9 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
;
; GFX12-LABEL: name: struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr2, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -466,8 +465,10 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (i16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (i16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.i16(i16 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
index f418ec0d41c55..b4f2d503f67a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
@@ -5,23 +5,6 @@
; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -160,24 +143,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0(ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
@@ -336,23 +301,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset.base, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
index a568c9b324320..413afff6d3b20 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
@@ -6,23 +6,6 @@
; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -37,8 +20,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
@@ -215,24 +199,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0(<4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
@@ -248,8 +214,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; UNPACKED: bb.1 (%ir-block.0):
@@ -466,23 +433,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset.base, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -497,8 +447,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; UNPACKED: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 540be44df1bfd..af6bfc3fc8559 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -698,9 +698,10 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-UNALIGNED-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v13
-; GFX12-UNALIGNED-NEXT: v_mov_b32_e32 v8, v2
-; GFX12-UNALIGNED-NEXT: v_mov_b32_e32 v2, v12
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX12-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-NOUNALIGNED-LABEL: v_load_constant_v12i8_align8:
@@ -721,9 +722,10 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-NOUNALIGNED-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v13
-; GFX12-NOUNALIGNED-NEXT: v_mov_b32_e32 v8, v2
-; GFX12-NOUNALIGNED-NEXT: v_mov_b32_e32 v2, v12
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX12-NOUNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_load_constant_v12i8_align8:
@@ -736,10 +738,12 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX1250-NEXT: v_dual_lshrrev_b32 v3, 24, v0 :: v_dual_lshrrev_b32 v5, 8, v1
; GFX1250-NEXT: v_dual_lshrrev_b32 v6, 16, v1 :: v_dual_lshrrev_b32 v7, 24, v1
; GFX1250-NEXT: v_dual_lshrrev_b32 v9, 8, v2 :: v_dual_lshrrev_b32 v10, 16, v2
-; GFX1250-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_lshrrev_b32 v11, 24, v2
+; GFX1250-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX1250-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX1250-NEXT: v_mov_b16_e32 v8.l, v2.l
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v1, v13
-; GFX1250-NEXT: v_mov_b32_e32 v2, v12
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX1250-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9-LABEL: v_load_constant_v12i8_align8:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
index 0a12cd16159b0..1e23985c424d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
@@ -33,11 +33,23 @@ define i16 @v_or_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_or_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_or_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_or_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_or_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_or_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_or_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_or_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = or i16 %num, %den
ret i16 %result
@@ -1109,6 +1121,3 @@ define <2 x i256> @v_or_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %or
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 9a43ae888f4e5..f4001c90113b4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -404,10 +404,8 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_saddsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index f5056d6247023..7b6cb87befba8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -404,10 +404,8 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_ssubsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 749fe014ff706..63964631573e2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -330,10 +330,8 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_uaddsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index 2e76abae2ff4f..6b32533b5781a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -324,10 +324,8 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_usubsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
index 41e66b9122086..7ae9ae02a961d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
@@ -33,11 +33,23 @@ define i16 @v_xor_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_xor_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_xor_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_xor_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_xor_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_xor_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_xor_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_xor_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = xor i16 %num, %den
ret i16 %result
@@ -1109,6 +1121,3 @@ define <2 x i256> @v_xor_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %xor
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 8c92ae4a47279..c4aec780a3628 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -15934,671 +15934,671 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v126, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v89, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v88, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v90, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v91, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v93, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v92, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v94, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v95, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v104, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v105, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v107, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v106, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v108, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v109, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v111, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v110, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v120, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v121, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v122, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v123, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v125, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v124, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v75.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v76.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v78.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v88.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v79.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v90.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v93.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v95.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v106.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v105.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v107.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v110.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v109.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v111.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v120.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v121.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v122.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v123.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v124.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v126, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v127, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v127, v126, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v125, v124, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v123, v122, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v121, v120, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v107, v105, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v106, v104, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v86, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v84, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v82, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v95, v94, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v111, v109, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v110, v108, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v93, v91, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v92, v90, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v89, v79, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v88, v78, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v39, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v36, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v11, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v74, v72, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v61, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v62, v60, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v59, v58, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v57, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v56, v46, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v45, v43, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v44, v42, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v41, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v33, v127, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v126, v124, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v125, v123, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v122, v121, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v120, v110, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v111, v109, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v108, v106, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v107, v105, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v104, v95, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v16, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v183, v181, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v179, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v178, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v167, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v164, v162, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v151, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v160, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v94, v92, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v93, v91, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v90, v88, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v89, v79, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v78, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v76, v74, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v72, v62, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v63, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v60, v59, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v15, 16, v14
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v17, 16, v16
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v19, 16, v18
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v16, v21, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v147, v145, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v146, v144, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v135, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v134, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v130, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v129, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v128, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v116, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v113, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v58, v56, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v57, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v46, v44, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v45, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v42, v41, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v40, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v183, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v180, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v179, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v176, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v17, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v18, v20, 16, v19
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v19, v22, 16, v21
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v20, v24, 16, v23
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v26, 16, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v99, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v98, v96, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v87, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v84, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v81, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v166, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v165, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v162, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v161, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v150, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v148, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v147, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v144, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v135, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v132, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v22, v23, 16, v22
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v23, v25, 16, v24
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v24, v27, 16, v26
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v25, v29, 16, v28
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v26, v31, 16, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v67, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v55, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v54, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v49, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v37, v39, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v35, v38, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v33, v36, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v32, v34, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v130, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v129, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v118, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v117, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v115, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v32, v112, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v33, v102, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v34, v101, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v99, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v97, v87, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v27, v28, 16, v27
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v28, v30, 16, v29
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v48, 16, v31
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v35, 16, v37
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v32, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v32, 16, v31
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v34, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v36, 16, v35
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr126_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr124_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr123_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: .LBB14_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v127.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v125.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v123.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v126.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v124.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v85.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v122.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v121.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v80.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v120.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v111.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v110.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v69.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v108.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v107.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v106.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v95.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v54.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v105.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v104.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v92.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v90.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v89.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v48.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v94.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v53.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v88.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v91.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v74.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v33.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v72.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v63.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v62.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v127.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v126.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v125.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v4.h
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v79.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v78.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v122.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v111.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v76.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v46.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v108.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v41.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v182.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v180.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v107.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v104.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v91.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v61.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v124.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v123.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v90.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v89.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v58.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v57.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v167.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v164.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v121.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v120.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v78.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v73.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v47.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v110.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v161.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v149.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v60.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v43.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v106.l
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v146.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v144.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v105.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v47.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v135.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v40.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v95.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v94.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v134.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v131.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v42.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v181.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v128.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v116.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v92.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v177.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v88.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v79.l
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v165.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v77.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v76.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v163.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v163.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v98.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v87.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v84.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v74.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v150.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v147.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v145.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v62.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v61.l
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v81.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v144.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v80.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v135.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v148.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v147.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v59.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v132.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v56.l
; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v53.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v114.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.h, v17.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v44.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v130.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v40.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.h, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v119.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v38.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v36.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v101.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v100.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v20.l, v19.h
; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v178.l
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v103.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v167.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v166.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l
+; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v101.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v34.l, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.l, v31.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v33.l, v31.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.h, v31.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.h, v22.l
; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v160.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v96.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v86.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v85.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v148.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v25.l, v24.h
; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v133.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v67.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v65.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v9.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v117.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v11.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v12.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v30.l, v27.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v28.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v112.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v102.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v30.l
; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v48.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v103.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v15.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.h, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.l, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v34.l, v32.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v17.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l
@@ -20586,21 +20586,21 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
@@ -20652,8 +20652,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v182, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v150, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s18, s19, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v115, 0xc0c0004
@@ -20720,7 +20720,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v162, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
@@ -20728,17 +20728,17 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v145, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v146, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v18
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v129, v118, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v19, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v144, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v145, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v132, v130, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
@@ -20901,11 +20901,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v162
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v163
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v160
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v150
; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v9
@@ -20915,7 +20915,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s6, s5, v10
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, s4, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v10
@@ -20924,7 +20924,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
@@ -20960,8 +20960,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v19
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v20, v21
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v147
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v145
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v132
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 3, v129
@@ -20969,7 +20969,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 3, v119
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 3, v114
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v131, 0xc0c0004
@@ -51944,671 +51944,671 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v126, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v89, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v88, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v90, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v91, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v93, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v92, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v94, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v95, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v104, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v105, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v107, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v106, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v108, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v109, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v111, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v110, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v120, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v121, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v122, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v123, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v125, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v124, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v75.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v76.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v78.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v88.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v79.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v90.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v93.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v95.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v106.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v105.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v107.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v110.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v109.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v111.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v120.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v121.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v122.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v123.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v124.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v126, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v127, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v127, v126, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v125, v124, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v123, v122, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v121, v120, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v107, v105, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v106, v104, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v86, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v84, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v82, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v95, v94, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v111, v109, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v110, v108, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v93, v91, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v92, v90, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v89, v79, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v88, v78, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v39, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v36, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v11, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v74, v72, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v61, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v62, v60, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v59, v58, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v57, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v56, v46, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v45, v43, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v44, v42, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v41, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v33, v127, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v126, v124, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v125, v123, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v122, v121, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v120, v110, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v111, v109, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v108, v106, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v107, v105, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v104, v95, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v16, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v183, v181, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v179, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v178, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v167, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v164, v162, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v151, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v160, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v94, v92, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v93, v91, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v90, v88, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v89, v79, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v78, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v76, v74, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v72, v62, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v63, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v60, v59, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v15, 16, v14
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v17, 16, v16
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v19, 16, v18
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v16, v21, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v147, v145, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v146, v144, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v135, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v134, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v130, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v129, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v128, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v116, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v113, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v58, v56, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v57, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v46, v44, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v45, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v42, v41, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v40, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v183, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v180, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v179, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v176, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v17, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v18, v20, 16, v19
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v19, v22, 16, v21
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v20, v24, 16, v23
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v26, 16, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v99, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v98, v96, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v87, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v84, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v81, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v166, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v165, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v162, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v161, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v150, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v148, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v147, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v144, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v135, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v132, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v22, v23, 16, v22
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v23, v25, 16, v24
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v24, v27, 16, v26
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v25, v29, 16, v28
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v26, v31, 16, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v67, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v55, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v54, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v49, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v37, v39, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v35, v38, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v33, v36, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v32, v34, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v130, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v129, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v118, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v117, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v115, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v32, v112, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v33, v102, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v34, v101, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v99, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v97, v87, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v27, v28, 16, v27
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v28, v30, 16, v29
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v48, 16, v31
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v35, 16, v37
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v32, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v32, 16, v31
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v34, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v36, 16, v35
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr126_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr124_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr123_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: .LBB38_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v127.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v125.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v123.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v126.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v124.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v85.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v122.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v121.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v80.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v120.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v111.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v110.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v69.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v108.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v107.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v106.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v95.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v54.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v105.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v104.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v92.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v90.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v89.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v48.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v94.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v53.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v88.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v91.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v74.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v33.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v72.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v63.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v62.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v127.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v126.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v125.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v4.h
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v79.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v78.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v122.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v111.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v76.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v46.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v108.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v41.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v182.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v180.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v107.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v104.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v91.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v61.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v124.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v123.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v90.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v89.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v58.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v57.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v167.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v164.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v121.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v120.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v78.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v73.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v47.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v110.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v161.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v149.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v60.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v43.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v106.l
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v146.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v144.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v105.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v47.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v135.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v40.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v95.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v94.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v134.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v131.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v42.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v181.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v128.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v116.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v92.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v177.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v88.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v79.l
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v165.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v77.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v76.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v163.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v163.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v98.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v87.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v84.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v74.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v150.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v147.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v145.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v62.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v61.l
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v81.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v144.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v80.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v135.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v148.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v147.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v59.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v132.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v56.l
; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v53.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v114.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.h, v17.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v44.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v130.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v40.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.h, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v119.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v38.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v36.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v101.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v100.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v20.l, v19.h
; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v178.l
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v103.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v167.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v166.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l
+; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v101.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v34.l, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.l, v31.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v33.l, v31.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.h, v31.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.h, v22.l
; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v160.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v96.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v86.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v85.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v148.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v25.l, v24.h
; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v133.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v67.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v65.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v9.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v117.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v11.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v12.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v30.l, v27.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v28.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v112.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v102.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v30.l
; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v48.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v103.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v15.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.h, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.l, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v34.l, v32.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v17.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l
@@ -56596,21 +56596,21 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
@@ -56662,8 +56662,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v182, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v150, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s18, s19, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v115, 0xc0c0004
@@ -56730,7 +56730,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v162, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
@@ -56738,17 +56738,17 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v145, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v146, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v18
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v129, v118, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v19, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v144, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v145, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v132, v130, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
@@ -56911,11 +56911,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v162
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v163
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v160
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v150
; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v9
@@ -56925,7 +56925,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s6, s5, v10
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, s4, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v10
@@ -56934,7 +56934,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
@@ -56970,8 +56970,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v19
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v20, v21
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v147
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v145
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v132
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 3, v129
@@ -56979,7 +56979,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 3, v119
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 3, v114
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v131, 0xc0c0004
@@ -85959,671 +85959,671 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v126, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v89, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v88, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v90, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v91, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v93, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v92, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v94, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v95, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v104, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v105, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v107, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v106, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v108, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v109, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v111, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v110, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v120, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v121, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v122, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v123, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v125, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v124, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v75.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v76.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v78.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v88.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v79.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v90.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v93.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v95.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v106.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v105.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v107.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v110.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v109.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v111.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v120.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v121.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v122.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v123.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v124.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v126, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v127, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB58_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v127, v126, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v125, v124, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v123, v122, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v121, v120, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v107, v105, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v106, v104, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v86, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v84, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v82, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v95, v94, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v111, v109, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v110, v108, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v93, v91, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v92, v90, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v89, v79, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v88, v78, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v39, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v36, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v11, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v74, v72, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v61, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v62, v60, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v59, v58, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v57, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v56, v46, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v45, v43, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v44, v42, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v41, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v33, v127, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v126, v124, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v125, v123, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v122, v121, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v120, v110, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v111, v109, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v108, v106, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v107, v105, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v104, v95, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v16, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v183, v181, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v179, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v178, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v167, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v164, v162, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v151, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v160, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v94, v92, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v93, v91, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v90, v88, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v89, v79, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v78, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v76, v74, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v72, v62, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v63, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v60, v59, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v15, 16, v14
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v17, 16, v16
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v19, 16, v18
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v16, v21, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v147, v145, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v146, v144, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v135, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v134, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v130, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v129, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v128, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v116, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v113, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v58, v56, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v57, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v46, v44, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v45, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v42, v41, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v40, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v183, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v180, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v179, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v176, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v17, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v18, v20, 16, v19
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v19, v22, 16, v21
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v20, v24, 16, v23
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v26, 16, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v99, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v98, v96, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v87, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v84, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v81, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v166, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v165, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v162, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v161, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v150, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v148, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v147, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v144, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v135, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v132, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v22, v23, 16, v22
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v23, v25, 16, v24
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v24, v27, 16, v26
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v25, v29, 16, v28
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v26, v31, 16, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v67, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v55, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v54, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v49, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v37, v39, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v35, v38, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v33, v36, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v32, v34, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v130, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v129, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v118, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v117, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v115, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v32, v112, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v33, v102, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v34, v101, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v99, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v97, v87, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v27, v28, 16, v27
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v28, v30, 16, v29
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v48, 16, v31
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v35, 16, v37
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v32, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v32, 16, v31
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v34, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v36, 16, v35
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr126_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr124_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr123_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: .LBB58_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB58_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v127.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v125.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v123.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v126.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v124.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v85.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v122.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v121.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v80.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v120.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v111.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v110.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v69.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v108.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v107.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v106.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v95.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v54.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v105.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v104.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v92.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v90.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v89.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v48.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v94.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v53.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v88.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v91.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v74.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v33.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v72.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v63.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v62.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v127.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v126.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v125.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v4.h
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v79.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v78.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v122.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v111.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v76.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v46.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v108.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v41.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v182.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v180.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v107.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v104.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v91.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v61.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v124.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v123.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v90.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v89.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v58.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v57.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v167.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v164.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v121.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v120.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v78.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v73.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v47.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v110.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v161.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v149.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v60.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v43.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v106.l
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v146.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v144.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v105.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v47.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v135.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v40.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v95.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v94.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v134.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v131.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v42.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v181.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v128.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v116.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v92.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v177.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v88.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v79.l
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v165.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v77.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v76.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v163.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v163.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v98.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v87.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v84.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v74.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v150.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v147.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v145.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v62.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v61.l
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v81.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v144.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v80.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v135.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v148.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v147.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v59.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v132.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v56.l
; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v53.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v114.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.h, v17.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v44.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v130.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v40.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.h, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v119.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v38.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v36.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v101.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v100.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v20.l, v19.h
; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v178.l
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v103.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v167.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v166.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l
+; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v101.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v34.l, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.l, v31.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v33.l, v31.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.h, v31.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.h, v22.l
; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v160.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v96.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v86.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v85.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v148.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v25.l, v24.h
; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v133.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v67.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v65.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v9.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v117.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v11.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v12.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v30.l, v27.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v28.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v112.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v102.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v30.l
; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v48.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v103.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v15.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.h, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.l, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v34.l, v32.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v17.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l
@@ -90611,21 +90611,21 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
@@ -90677,8 +90677,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v182, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v150, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s18, s19, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v115, 0xc0c0004
@@ -90745,7 +90745,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v162, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
@@ -90753,17 +90753,17 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v145, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v146, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v18
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v129, v118, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v19, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v144, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v145, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v132, v130, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
@@ -90926,11 +90926,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v162
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v163
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v160
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v150
; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v9
@@ -90940,7 +90940,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s6, s5, v10
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, s4, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v10
@@ -90949,7 +90949,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
@@ -90985,8 +90985,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v19
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v20, v21
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v147
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v145
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v132
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 3, v129
@@ -90994,7 +90994,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 3, v119
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 3, v114
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v131, 0xc0c0004
@@ -118985,671 +118985,671 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v126, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v89, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v88, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v90, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v91, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v93, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v92, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v94, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v95, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v104, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v105, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v107, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v106, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v108, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v109, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v111, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v110, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v120, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v121, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v122, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v123, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v125, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v124, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v75.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v76.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v78.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v88.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v79.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v90.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v93.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v95.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v106.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v105.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v107.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v110.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v109.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v111.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v120.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v121.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v122.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v123.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v124.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v126, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v127, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB74_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v127, v126, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v125, v124, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v123, v122, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v121, v120, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v107, v105, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v106, v104, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v86, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v84, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v82, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v95, v94, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v111, v109, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v110, v108, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v93, v91, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v92, v90, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v89, v79, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v88, v78, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v39, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v36, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v11, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v74, v72, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v61, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v62, v60, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v59, v58, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v57, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v56, v46, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v45, v43, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v44, v42, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v41, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v33, v127, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v126, v124, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v125, v123, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v122, v121, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v120, v110, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v111, v109, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v108, v106, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v107, v105, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v104, v95, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v16, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v183, v181, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v179, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v178, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v167, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v164, v162, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v151, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v160, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v94, v92, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v93, v91, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v90, v88, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v89, v79, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v78, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v76, v74, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v75, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v72, v62, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v63, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v60, v59, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v15, 16, v14
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v17, 16, v16
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v19, 16, v18
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v16, v21, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v147, v145, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v146, v144, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v135, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v134, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v130, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v129, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v128, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v116, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v113, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v58, v56, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v57, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v46, v44, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v45, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v42, v41, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v40, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v183, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v180, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v179, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v176, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v17, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v18, v20, 16, v19
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v19, v22, 16, v21
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v20, v24, 16, v23
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v26, 16, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v99, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v98, v96, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v87, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v84, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v81, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v166, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v165, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v162, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v161, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v150, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v148, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v147, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v144, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v135, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v132, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v22, v23, 16, v22
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v23, v25, 16, v24
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v24, v27, 16, v26
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v25, v29, 16, v28
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v26, v31, 16, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v67, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v55, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v54, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v49, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v37, v39, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v35, v38, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v33, v36, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v32, v34, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v130, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v129, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v118, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v117, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v115, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v32, v112, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v33, v102, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v34, v101, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v99, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v97, v87, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v27, v28, 16, v27
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v28, v30, 16, v29
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v48, 16, v31
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v35, 16, v37
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v32, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v29, v32, 16, v31
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v30, v34, 16, v33
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v31, v36, 16, v35
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr126_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr124_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr123_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: .LBB74_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB74_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v127.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v125.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v123.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v126.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v124.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v85.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v122.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v121.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v80.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v120.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v111.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v110.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v69.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v108.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v107.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v106.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v95.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v54.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v105.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v104.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v92.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v90.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v89.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v48.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v94.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v53.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v88.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v91.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v74.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v33.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v72.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v63.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v62.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v127.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v126.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v125.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v4.h
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v79.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v78.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v122.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v111.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v76.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v46.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v109.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v108.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v41.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v182.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v180.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v107.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v104.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v93.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v91.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v61.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v124.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v123.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v90.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v89.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v58.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v57.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v167.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v164.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v121.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v120.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v78.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v75.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v73.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.h, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v47.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v110.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v161.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v149.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v60.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v43.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v106.l
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v146.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v144.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v105.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v47.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v135.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v40.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v95.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v94.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v134.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v131.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v45.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v42.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v11.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v181.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v128.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v116.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v92.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v183.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v177.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v88.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v79.l
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v165.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v77.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v76.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v163.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v163.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v98.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v87.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v84.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v74.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v150.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v147.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v145.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v62.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v61.l
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v81.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v144.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v15.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v80.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v135.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v148.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v147.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v59.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v132.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v16.h
; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v56.l
; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v53.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v114.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.h, v17.l
; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v44.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v28.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v130.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v129.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v40.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.h, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v119.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v38.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v36.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, v101.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v100.l
; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v20.l, v19.h
; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v178.l
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v21.l, v20.h
; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v103.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v167.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v166.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l
+; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v22.l, v21.h
; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v101.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v34.l, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.l, v31.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v33.l, v31.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v33.h, v31.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.h, v22.l
; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v160.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v96.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.h, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v86.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v85.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v148.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.h, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v25.l, v24.h
; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v133.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v25.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v67.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v26.h
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v65.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v9.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.h, v27.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v117.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v11.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v12.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v30.l, v27.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v28.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v112.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v102.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v30.l
; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v48.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v14.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v103.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v15.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.h, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v30.h, v30.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v32.l, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v34.l, v32.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v17.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l
@@ -123637,21 +123637,21 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
@@ -123703,8 +123703,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v182, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v149, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v165, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v150, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s18, s19, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v115, 0xc0c0004
@@ -123771,7 +123771,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v162, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
@@ -123779,17 +123779,17 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v145, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v146, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v18
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v160, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v147, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v129, v118, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v19, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v144, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v145, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v132, v130, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
@@ -123952,11 +123952,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v162
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v163
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v160
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v150
; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v9
@@ -123966,7 +123966,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s6, s5, v10
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, s4, v180, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v10
@@ -123975,7 +123975,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v149, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
@@ -124011,8 +124011,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v19
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v20, v21
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v147
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v145
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v132
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 3, v129
@@ -124020,7 +124020,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 3, v119
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 3, v114
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v131, 0xc0c0004
@@ -143935,771 +143935,735 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v128i8_to_v64bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:488
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:484
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:480
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:476
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:472
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:468
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:464
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:460
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:456
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:452
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Spill
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:424
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:448
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:420
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:444
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:416
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:440
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:412
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:436
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:408
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:432
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:404
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:428
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:400
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v72, s32 offset:424
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v73, s32 offset:420
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v74, s32 offset:416
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v75, s32 offset:412
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v76, s32 offset:408
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v77, s32 offset:404
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v78, s32 offset:400
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v79, s32 offset:396
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:248
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v6, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_b32 v12, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v6, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v7, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v7, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v8, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v8, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v9, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v9, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v10, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v10, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v11, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v11, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v28.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v13.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.h, 8, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.h, 8, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v6.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v7.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v9.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.h, 8, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v10.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v11.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB88_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v81, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v85, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v37.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v54.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v84, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v82.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v86.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v96.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v49.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v97.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v160, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v129, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v84.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v83.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v182, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v131, v116, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v87.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v85.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v41, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v146, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v87.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v96.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v115, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v150, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v119, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v162, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v130, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v167, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v132, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v38, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v135, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v40, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v151, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v64.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v166, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v69.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v83, v70, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v183, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v71.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v36, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v99.l
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v67.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v50.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v64, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v68.h
+; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v65.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v86, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v100.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v103.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v43, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v99.h
+; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v98.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v45, v44, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
-; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v101.l
+; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v101.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v46, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v102.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v102.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v56, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v80.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v81.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v97.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v98.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: .LBB88_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB88_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v70.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v66.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v46.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v183.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v103.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v102.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v69.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v102.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v68.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v65.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v100.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v101.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v101.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v44.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v67.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v43.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v100.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v99.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v52.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v98.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v99.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v40.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v98.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v178.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v177.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v166.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v163.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v147.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v135.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v144.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v132.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v146.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v118.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v119.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v129.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v112.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v55.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.l, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v41.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.h, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v96.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v176.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v53.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v96.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v165.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v85.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v86.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v84.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v83.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v82.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v84.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v80.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v81.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v83.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v81.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v65.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v33.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v33.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
@@ -144707,32 +144671,16 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v32.h
; GFX11-TRUE16-NEXT: .LBB88_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:392
-; GFX11-TRUE16-NEXT: scratch_load_b32 v79, off, s32 offset:396
-; GFX11-TRUE16-NEXT: scratch_load_b32 v78, off, s32 offset:400
-; GFX11-TRUE16-NEXT: scratch_load_b32 v77, off, s32 offset:404
-; GFX11-TRUE16-NEXT: scratch_load_b32 v76, off, s32 offset:408
-; GFX11-TRUE16-NEXT: scratch_load_b32 v75, off, s32 offset:412
-; GFX11-TRUE16-NEXT: scratch_load_b32 v74, off, s32 offset:416
-; GFX11-TRUE16-NEXT: scratch_load_b32 v73, off, s32 offset:420
-; GFX11-TRUE16-NEXT: scratch_load_b32 v72, off, s32 offset:424
-; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:428
-; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:432
-; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:436
-; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:440
-; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:444
-; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:448
-; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:452
-; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:456
-; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:460
-; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:464
-; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:468
-; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:472
-; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:476
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:480
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:484
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:488
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Reload
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:396
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:400
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:404
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:408
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:412
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:416
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:420
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:424
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -149052,119 +149000,119 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
@@ -149172,9 +149120,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB89_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v181
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s8, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v180
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s42, 8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v176
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
@@ -149185,49 +149133,49 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s76 :: v_dual_lshlrev_b32 v15, 8, v177
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v178
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s88, s88, s89
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s88
; GFX11-TRUE16-NEXT: s_and_b32 s88, s74, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s76, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v180
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v181
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, s88, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v164
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v150
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v162
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v148
; GFX11-TRUE16-NEXT: s_or_b32 s76, s77, s76
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s12, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s57, 8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: s_or_b32 s79, s88, s89
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v8.l
@@ -149235,89 +149183,89 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v167
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 8, v147
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v160
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v161
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s78, s79
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v165
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v163
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v8.l
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v151, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v160, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v39, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v162, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v165, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v161, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v81, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v81, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v84, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v96, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v65, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v99, v54, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v65, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v101, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v67, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v112, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v69, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v71, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v102, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v112, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v115, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v80
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v37
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v116, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v38
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v84
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v50
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v132
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v87
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v98
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v52
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v102
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v113
; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v8, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v131, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v134, v114, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v26, v27
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v98
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v64
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v83
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v101
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v86
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v130
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v67
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v133
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v69
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v115
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v66
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v118
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v68
; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v8, v27
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v28, v29
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v118
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v86
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v133, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v129
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v97
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v144, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v134
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v113
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v145
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v116
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v28
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v97
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v144
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v100
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v149
; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v8, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v119
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v130
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v128
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v85
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v41, v29, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v132
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v145, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v135
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v150, v128, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v183, v8, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v40, v31
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v135, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v146, v117, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
@@ -149333,117 +149281,117 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB89_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v119
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v144
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v128
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v150
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v135, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v117, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v129
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v133
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v97, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v86, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v130
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v69, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v87
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v86, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v132
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v80
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v84, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v83
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v119
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v112
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v99
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v67
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v55
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v96
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v51
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v55
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v49
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v165
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v149, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v167
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v147, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v164
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v163, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v180
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -149451,116 +149399,116 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v5
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v69, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v67, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v80, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s22, s23, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v83, s2, s3, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v84, s18, s19, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v82, s2, s3, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v83, s18, s19, v1
; GFX11-TRUE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-TRUE16-NEXT: s_add_i32 s16, s16, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v3
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v85, s16, s17, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v87, s0, s1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v84, s16, s17, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v86, s0, s1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v80, 0x300, v80
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v82
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v83, 0x300, v83
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v84
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v96, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v87
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v83.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v84.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v96.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v82.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v83.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v87.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v80.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v81.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v71.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v68.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v65.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v65.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v54.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v52.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v39.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v86.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v66.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v64.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v52.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v66.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v68.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v100.l
; GFX11-TRUE16-NEXT: .LBB89_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
@@ -170347,771 +170295,735 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v128i8_to_v64f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:488
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:484
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:480
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:476
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Spill
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:424
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:472
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:420
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:468
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:416
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:464
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:412
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:460
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:408
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:456
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:404
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:452
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:400
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:448
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:444
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:440
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:436
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:432
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:428
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v72, s32 offset:424
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v73, s32 offset:420
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v74, s32 offset:416
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v75, s32 offset:412
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v76, s32 offset:408
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v77, s32 offset:404
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v78, s32 offset:400
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v79, s32 offset:396
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:248
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v6, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_b32 v12, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v6, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v7, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v7, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v8, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v8, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v9, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v9, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v10, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v10, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v11, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v11, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v28.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v13.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.h, 8, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.h, 8, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v6.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v7.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v9.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.h, 8, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v10.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v11.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB92_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v81, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v85, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v37.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v54.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v84, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v82.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v86.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v96.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v49.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v97.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v160, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v129, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v84.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v83.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v182, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v131, v116, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v87.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v85.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v41, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v146, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v87.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v96.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v115, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v150, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v119, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v162, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v130, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v167, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v132, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v38, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v135, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v40, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v151, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v64.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v166, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v69.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v83, v70, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v183, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v71.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v36, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v99.l
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v67.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v50.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v64, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v68.h
+; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v65.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v86, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v100.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v103.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v43, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v99.h
+; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v98.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v45, v44, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
-; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v101.l
+; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v101.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v46, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v102.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v102.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v56, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v80.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v81.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v97.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v98.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: .LBB92_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB92_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v70.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v66.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v46.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v183.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v103.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v102.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v69.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v102.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v68.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v65.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v100.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v101.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v101.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v44.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v67.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v43.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v100.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v99.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v52.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v98.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v99.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v40.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v98.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v178.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v177.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v166.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v163.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v147.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v135.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v144.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v132.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v146.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v118.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v119.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v129.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v112.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v55.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.l, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v41.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.h, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v96.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v176.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v53.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v96.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v165.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v85.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v86.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v84.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v83.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v82.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v84.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v80.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v81.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v83.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v81.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v65.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v33.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v33.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
@@ -171119,32 +171031,16 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v32.h
; GFX11-TRUE16-NEXT: .LBB92_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:392
-; GFX11-TRUE16-NEXT: scratch_load_b32 v79, off, s32 offset:396
-; GFX11-TRUE16-NEXT: scratch_load_b32 v78, off, s32 offset:400
-; GFX11-TRUE16-NEXT: scratch_load_b32 v77, off, s32 offset:404
-; GFX11-TRUE16-NEXT: scratch_load_b32 v76, off, s32 offset:408
-; GFX11-TRUE16-NEXT: scratch_load_b32 v75, off, s32 offset:412
-; GFX11-TRUE16-NEXT: scratch_load_b32 v74, off, s32 offset:416
-; GFX11-TRUE16-NEXT: scratch_load_b32 v73, off, s32 offset:420
-; GFX11-TRUE16-NEXT: scratch_load_b32 v72, off, s32 offset:424
-; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:428
-; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:432
-; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:436
-; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:440
-; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:444
-; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:448
-; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:452
-; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:456
-; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:460
-; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:464
-; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:468
-; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:472
-; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:476
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:480
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:484
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:488
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Reload
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:396
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:400
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:404
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:408
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:412
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:416
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:420
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:424
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -175469,119 +175365,119 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
@@ -175589,9 +175485,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB93_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v181
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s8, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v180
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s42, 8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v176
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
@@ -175602,49 +175498,49 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s76 :: v_dual_lshlrev_b32 v15, 8, v177
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v178
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s88, s88, s89
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s88
; GFX11-TRUE16-NEXT: s_and_b32 s88, s74, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s76, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v180
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v181
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, s88, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v164
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v150
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v162
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v148
; GFX11-TRUE16-NEXT: s_or_b32 s76, s77, s76
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s12, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s57, 8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: s_or_b32 s79, s88, s89
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v8.l
@@ -175652,89 +175548,89 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v167
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 8, v147
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v160
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v161
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s78, s79
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v165
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v163
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v8.l
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v151, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v160, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v39, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v162, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v165, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v161, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v81, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v81, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v84, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v96, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v65, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v99, v54, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v65, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v101, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v67, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v112, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v69, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v71, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v102, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v112, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v115, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v80
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v37
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v116, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v38
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v84
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v50
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v132
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v87
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v98
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v52
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v102
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v113
; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v8, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v131, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v134, v114, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v26, v27
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v98
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v64
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v83
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v101
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v86
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v130
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v67
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v133
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v69
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v115
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v66
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v118
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v68
; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v8, v27
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v28, v29
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v118
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v86
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v133, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v129
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v97
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v144, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v134
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v113
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v145
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v116
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v28
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v97
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v144
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v100
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v149
; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v8, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v119
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v130
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v128
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v85
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v41, v29, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v132
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v145, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v135
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v150, v128, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v183, v8, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v40, v31
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v135, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v146, v117, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
@@ -175750,117 +175646,117 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB93_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v119
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v144
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v128
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v150
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v135, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v117, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v129
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v133
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v97, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v86, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v130
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v69, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v87
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v86, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v132
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v80
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v84, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v83
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v119
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v112
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v99
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v67
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v55
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v96
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v51
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v55
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v49
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v165
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v149, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v167
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v147, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v164
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v163, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v180
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -175868,116 +175764,116 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v5
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v69, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v67, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v80, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s22, s23, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v83, s2, s3, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v84, s18, s19, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v82, s2, s3, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v83, s18, s19, v1
; GFX11-TRUE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-TRUE16-NEXT: s_add_i32 s16, s16, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v3
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v85, s16, s17, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v87, s0, s1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v84, s16, s17, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v86, s0, s1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v80, 0x300, v80
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v82
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v83, 0x300, v83
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v84
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v96, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v87
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v83.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v84.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v96.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v82.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v83.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v87.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v80.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v81.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v71.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v68.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v65.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v65.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v54.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v52.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v39.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v86.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v66.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v64.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v52.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v66.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v68.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v100.l
; GFX11-TRUE16-NEXT: .LBB93_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
@@ -191175,771 +191071,735 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v128i8_to_v64i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:488
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:484
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:480
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:476
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:472
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Spill
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:424
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:468
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:420
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:464
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:416
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:460
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:412
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:456
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:408
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:452
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:404
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:448
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:400
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:444
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:396
; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:440
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:436
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:432
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:428
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v72, s32 offset:424
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v73, s32 offset:420
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v74, s32 offset:416
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v75, s32 offset:412
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v76, s32 offset:408
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v77, s32 offset:404
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v78, s32 offset:400
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v79, s32 offset:396
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:392
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:384
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v47, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:368
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v42, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:332
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v45, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v40, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:248
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v6, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_b32 v12, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v6, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v7, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v7, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v8, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v8, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v9, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v9, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v10, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v10, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v11, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v11, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v28.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v13.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v102.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.h, 8, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.h, 8, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v6.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v7.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v9.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.h, 8, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v10.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v11.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v65, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v81, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v85, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v37.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v54.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v84, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v82.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v86.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v96.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v49.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v97.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v160, v134, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v129, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v84.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v83.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v182, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v131, v116, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v87.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v85.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v41, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v146, v133, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v87.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v96.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v115, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v150, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v119, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v162, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v130, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v167, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v132, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v38, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v161, v135, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v40, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v151, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v64.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v166, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v69.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v83, v70, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v183, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v71.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v36, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v99.l
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v67.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v50.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v64, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v68.h
+; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v65.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v86, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v100.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v103.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v43, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v99.h
+; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v98.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v45, v44, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
-; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v101.l
+; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v101.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v46, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
-; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
-; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v102.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v102.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v56, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v80.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v81.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v97.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v98.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v56.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v70.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v66.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v46.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v183.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v103.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v102.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v69.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v102.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v68.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v65.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v100.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v101.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v101.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v44.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v67.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v43.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v100.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v99.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v52.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v98.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v99.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v40.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v164.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v98.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v178.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v177.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v166.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v163.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v145.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v162.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v147.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v135.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v144.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v132.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v146.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v118.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v119.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v129.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v115.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v112.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v55.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v97.l, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v41.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.h, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v96.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v176.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v53.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v96.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v87.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v165.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v85.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v86.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v84.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v83.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v134.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v82.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v84.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v80.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v81.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v128.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v83.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v81.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v65.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v32.l
+; GFX11-TRUE16-NEXT: v_and_b16 v33.h, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v33.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v34.l, v33.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v1.h
@@ -191947,32 +191807,16 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v32.h
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_clause 0x18 ; 100-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:392
-; GFX11-TRUE16-NEXT: scratch_load_b32 v79, off, s32 offset:396
-; GFX11-TRUE16-NEXT: scratch_load_b32 v78, off, s32 offset:400
-; GFX11-TRUE16-NEXT: scratch_load_b32 v77, off, s32 offset:404
-; GFX11-TRUE16-NEXT: scratch_load_b32 v76, off, s32 offset:408
-; GFX11-TRUE16-NEXT: scratch_load_b32 v75, off, s32 offset:412
-; GFX11-TRUE16-NEXT: scratch_load_b32 v74, off, s32 offset:416
-; GFX11-TRUE16-NEXT: scratch_load_b32 v73, off, s32 offset:420
-; GFX11-TRUE16-NEXT: scratch_load_b32 v72, off, s32 offset:424
-; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:428
-; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:432
-; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:436
-; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:440
-; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:444
-; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:448
-; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:452
-; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:456
-; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:460
-; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:464
-; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:468
-; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:472
-; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:476
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:480
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:484
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:488
+; GFX11-TRUE16-NEXT: s_clause 0x8 ; 36-byte Folded Reload
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:392
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:396
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:400
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:404
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:408
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:412
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:416
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:420
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:424
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -196297,119 +196141,119 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:304
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:288
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:272
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:300
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:284
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:268
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
@@ -196417,9 +196261,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v181
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s8, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v180
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s42, 8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v176
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
@@ -196430,49 +196274,49 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s76 :: v_dual_lshlrev_b32 v15, 8, v177
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v178
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s88, s88, s89
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s88
; GFX11-TRUE16-NEXT: s_and_b32 s88, s74, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s76, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v180
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v181
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, s88, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v164
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v150
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v162
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 8, v148
; GFX11-TRUE16-NEXT: s_or_b32 s76, s77, s76
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s88, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s12, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s88, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s89, s57, 8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: s_or_b32 s79, s88, s89
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v8.l
@@ -196480,89 +196324,89 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xff, v167
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 8, v147
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v160
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xff, v161
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s78, s79
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v165
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 8, v163
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v182, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v8.l
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v151, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v160, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v166, v164, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v39, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v162, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v165, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v161, v148, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v81, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v49, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v81, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v51, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v84, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v55, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v96, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v65, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v99, v54, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v65, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v101, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v67, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v112, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v69, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v71, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v102, v53, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v112, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v115, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v80
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v37
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v116, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 8, v38
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v119, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v84
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v50
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v132
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 8, v87
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xff, v98
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v52
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v102
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xff, v113
; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v8, v25
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v131, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v134, v114, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v26, v27
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v98
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v64
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v83
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v101
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 8, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 8, v86
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v130
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v67
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v133
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v69
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v115
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v66
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xff, v118
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v68
; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v8, v27
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v28, v29
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v118
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v86
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v133, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v129
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 8, v97
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v144, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v134
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v113
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v145
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 8, v116
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v28
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v97
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v144
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v100
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v40, 0xff, v149
; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v8, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v119
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xff, v130
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v128
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xff, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 8, v85
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v41, v29, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v132
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v145, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 8, v135
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v150, v128, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v183, v8, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v40, v31
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v135, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v146, v117, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
@@ -196578,117 +196422,117 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v119
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v144
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v128
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v150
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v135, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v117, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v129
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v133
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v97, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v86, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v130
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v131
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v69, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v87
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v86, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v132
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v66, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v80
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v84, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v83
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v119
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v71, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v112
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v99
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v101
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v102
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v67
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v55
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v65
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v96
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v51
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v55
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v81
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v49
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v165
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v149, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v160
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v161
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v167
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v147, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v164
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v165, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v162
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v163, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v180
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v150, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -196696,116 +196540,116 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v181, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v176, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v5
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v69, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v67, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v80, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s22, s23, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v83, s2, s3, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v84, s18, s19, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v82, s2, s3, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v83, s18, s19, v1
; GFX11-TRUE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-TRUE16-NEXT: s_add_i32 s16, s16, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v3
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v85, s16, s17, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v87, s0, s1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v84, s16, s17, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v86, s0, s1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v80, 0x300, v80
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v82
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v83, 0x300, v83
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v84
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v96, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v85
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v87
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v83.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v84.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v96.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v84
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v82.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v83.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v87.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v80.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v81.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v71.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v68.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v65.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v65.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v55.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v54.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v52.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v39.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v86.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v66.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v64.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v52.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v66.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v97.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v68.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v100.l
; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 17fa6366bbca1..991071ac55bc5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -3797,8 +3797,8 @@ define <16 x i8> @bitcast_v4i32_to_v16i8(<4 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4i32_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -3808,7 +3808,7 @@ define <16 x i8> @bitcast_v4i32_to_v16i8(<4 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -3816,46 +3816,46 @@ define <16 x i8> @bitcast_v4i32_to_v16i8(<4 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB24_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v17
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i32_to_v16i8:
@@ -4164,74 +4164,150 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i32_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s18, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB25_3
-; GFX11-NEXT: .LBB25_2:
-; GFX11-NEXT: s_mov_b32 s18, -1
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: .LBB25_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s18, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s3, s3, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: .LBB25_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
-; GFX11-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s15
-; GFX11-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s13
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s12
-; GFX11-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v11, s4
-; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
-; GFX11-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i32_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB25_3
+; GFX11-TRUE16-NEXT: .LBB25_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s3, s3, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i32_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s18, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB25_3
+; GFX11-FAKE16-NEXT: .LBB25_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s18, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s3, s3, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-FAKE16-NEXT: .LBB25_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s12
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v11, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4559,10 +4635,10 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -4575,8 +4651,8 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -4586,10 +4662,10 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -4607,11 +4683,11 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -8558,8 +8634,8 @@ define <16 x i8> @bitcast_v4f32_to_v16i8(<4 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4f32_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -8569,7 +8645,7 @@ define <16 x i8> @bitcast_v4f32_to_v16i8(<4 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -8577,44 +8653,44 @@ define <16 x i8> @bitcast_v4f32_to_v16i8(<4 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v12, 1.0, v12
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v18, 1.0, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f32_to_v16i8:
@@ -8941,83 +9017,162 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, v17
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4f32_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s18, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB49_3
-; GFX11-NEXT: .LBB49_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: .LBB49_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v19, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v17, s3, 1.0
-; GFX11-NEXT: v_add_f32_e64 v16, s2, 1.0
-; GFX11-NEXT: v_add_f32_e64 v18, s0, 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v17
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB49_6
-; GFX11-NEXT: .LBB49_5:
-; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
-; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
-; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
-; GFX11-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
-; GFX11-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
-; GFX11-NEXT: v_mov_b32_e32 v13, s12
-; GFX11-NEXT: v_mov_b32_e32 v15, s9
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB49_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v18
-; GFX11-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-NEXT: v_mov_b32_e32 v8, v16
-; GFX11-NEXT: v_mov_b32_e32 v12, v17
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4f32_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, s3, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, s2, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4f32_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v19, s1, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v17, s3, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v16, s2, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v18, s0, 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, s12
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s9
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v19
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v17
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9345,10 +9500,10 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -9361,8 +9516,8 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -9372,10 +9527,10 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -9393,11 +9548,11 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -12912,8 +13067,8 @@ define <16 x i8> @bitcast_v2i64_to_v16i8(<2 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2i64_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -12923,7 +13078,7 @@ define <16 x i8> @bitcast_v2i64_to_v16i8(<2 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -12931,46 +13086,46 @@ define <16 x i8> @bitcast_v2i64_to_v16i8(<2 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_co_u32 v11, vcc_lo, v11, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i64_to_v16i8:
@@ -13279,101 +13434,177 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2i64_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s18, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB69_3
-; GFX11-NEXT: .LBB69_2:
-; GFX11-NEXT: s_mov_b32 s18, -1
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: .LBB69_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s18, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_u32 s0, s0, 3
-; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: s_add_u32 s2, s2, 3
-; GFX11-NEXT: s_addc_u32 s3, s3, 0
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: .LBB69_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
-; GFX11-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s15
-; GFX11-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s13
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s12
-; GFX11-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v11, s4
-; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
-; GFX11-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
-; GFX11-NEXT: s_setpc_b64 s[30:31]
- %cmp = icmp eq i32 %b, 0
- br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
- %a1 = add <2 x i64> %a, splat (i64 3)
- %a2 = bitcast <2 x i64> %a1 to <16 x i8>
- br label %end
-
-cmp.false:
- %a3 = bitcast <2 x i64> %a to <16 x i8>
- br label %end
-
-end:
- %phi = phi <16 x i8> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
- ret <16 x i8> %phi
-}
-
-define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
-; SI-LABEL: bitcast_v16i8_to_v2i64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v18, v2
-; SI-NEXT: v_mov_b32_e32 v17, v0
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v1
-; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v3
-; SI-NEXT: v_lshlrev_b32_e32 v20, 8, v5
+; GFX11-TRUE16-LABEL: bitcast_v2i64_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB69_3
+; GFX11-TRUE16-NEXT: .LBB69_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-TRUE16-NEXT: s_add_u32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, 0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2i64_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s18, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB69_3
+; GFX11-FAKE16-NEXT: .LBB69_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s18, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-FAKE16-NEXT: s_add_u32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s3, s3, 0
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-FAKE16-NEXT: .LBB69_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s12
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v11, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %cmp = icmp eq i32 %b, 0
+ br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+ %a1 = add <2 x i64> %a, splat (i64 3)
+ %a2 = bitcast <2 x i64> %a1 to <16 x i8>
+ br label %end
+
+cmp.false:
+ %a3 = bitcast <2 x i64> %a to <16 x i8>
+ br label %end
+
+end:
+ %phi = phi <16 x i8> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+ ret <16 x i8> %phi
+}
+
+define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
+; SI-LABEL: bitcast_v16i8_to_v2i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v18, v2
+; SI-NEXT: v_mov_b32_e32 v17, v0
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
+; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v20, 8, v5
; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v7
; SI-NEXT: v_lshlrev_b32_e32 v16, 8, v9
; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v11
@@ -13674,10 +13905,10 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -13690,8 +13921,8 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -13701,10 +13932,10 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -13722,11 +13953,11 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -16823,8 +17054,8 @@ define <16 x i8> @bitcast_v2f64_to_v16i8(<2 x double> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2f64_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -16834,7 +17065,7 @@ define <16 x i8> @bitcast_v2f64_to_v16i8(<2 x double> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -16842,43 +17073,43 @@ define <16 x i8> @bitcast_v2f64_to_v16i8(<2 x double> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB84_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], v[11:12], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], v[18:19], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8:
@@ -17198,80 +17429,156 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, v17
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2f64_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s14, s3, 24
-; GFX11-NEXT: s_lshr_b32 s13, s3, 16
-; GFX11-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-NEXT: s_lshr_b32 s16, s2, 16
-; GFX11-NEXT: s_lshr_b32 s15, s2, 8
-; GFX11-NEXT: s_lshr_b32 s11, s1, 24
-; GFX11-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-NEXT: s_lshr_b32 s18, s0, 16
-; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB85_3
-; GFX11-NEXT: .LBB85_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: .LBB85_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f64 v[16:17], s[2:3], 1.0
-; GFX11-NEXT: v_add_f64 v[18:19], s[0:1], 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB85_6
-; GFX11-NEXT: .LBB85_5:
-; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v17, s3
-; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v19, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, s15
-; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v14, s13
-; GFX11-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v6, s10
-; GFX11-NEXT: v_mov_b32_e32 v15, s14
-; GFX11-NEXT: v_mov_b32_e32 v13, s12
-; GFX11-NEXT: v_mov_b32_e32 v7, s11
-; GFX11-NEXT: v_mov_b32_e32 v5, s9
-; GFX11-NEXT: .LBB85_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v18
-; GFX11-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-NEXT: v_mov_b32_e32 v8, v16
-; GFX11-NEXT: v_mov_b32_e32 v12, v17
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v2f64_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB85_3
+; GFX11-TRUE16-NEXT: .LBB85_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], s[2:3], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB85_6
+; GFX11-TRUE16-NEXT: .LBB85_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, s15
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v14, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s6 :: v_dual_mov_b32 v16, s12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s14 :: v_dual_mov_b32 v6, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s9
+; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s18, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB85_3
+; GFX11-FAKE16-NEXT: .LBB85_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f64 v[16:17], s[2:3], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[18:19], s[0:1], 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-FAKE16-NEXT: s_branch .LBB85_6
+; GFX11-FAKE16-NEXT: .LBB85_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v17, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v19, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v14, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v6, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, s12
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s11
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s9
+; GFX11-FAKE16-NEXT: .LBB85_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v19
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v17
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -17599,10 +17906,10 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -17615,8 +17922,8 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -17626,10 +17933,10 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -17647,11 +17954,11 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -20421,8 +20728,8 @@ define <16 x i8> @bitcast_v8i16_to_v16i8(<8 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i16_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -20432,7 +20739,7 @@ define <16 x i8> @bitcast_v8i16_to_v16i8(<8 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -20440,46 +20747,46 @@ define <16 x i8> @bitcast_v8i16_to_v16i8(<8 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i16_to_v16i8:
@@ -20840,83 +21147,162 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v12, v17
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i16_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s18, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB97_3
-; GFX11-NEXT: .LBB97_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: .LBB97_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v19, s1, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v17, s3, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v16, s2, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v18, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v17
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB97_6
-; GFX11-NEXT: .LBB97_5:
-; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
-; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
-; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
-; GFX11-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
-; GFX11-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
-; GFX11-NEXT: v_mov_b32_e32 v13, s12
-; GFX11-NEXT: v_mov_b32_e32 v15, s9
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB97_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v18
-; GFX11-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-NEXT: v_mov_b32_e32 v8, v16
-; GFX11-NEXT: v_mov_b32_e32 v12, v17
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i16_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB97_3
+; GFX11-TRUE16-NEXT: .LBB97_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, s3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, s2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB97_6
+; GFX11-TRUE16-NEXT: .LBB97_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i16_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB97_3
+; GFX11-FAKE16-NEXT: .LBB97_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v19, s1, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v17, s3, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v16, s2, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v18, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-FAKE16-NEXT: s_branch .LBB97_6
+; GFX11-FAKE16-NEXT: .LBB97_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, s12
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s9
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB97_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v19
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v17
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -21265,10 +21651,10 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -21281,8 +21667,8 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -21292,10 +21678,10 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -21313,11 +21699,11 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -23491,8 +23877,8 @@ define <16 x i8> @bitcast_v8f16_to_v16i8(<8 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8f16_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -23502,7 +23888,7 @@ define <16 x i8> @bitcast_v8f16_to_v16i8(<8 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -23510,46 +23896,46 @@ define <16 x i8> @bitcast_v8f16_to_v16i8(<8 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB104_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f16_to_v16i8:
@@ -23930,83 +24316,162 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX9-NEXT: v_mov_b32_e32 v12, v17
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8f16_to_v16i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-NEXT: s_lshr_b32 s18, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB105_3
-; GFX11-NEXT: .LBB105_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: .LBB105_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s3 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v17
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB105_6
-; GFX11-NEXT: .LBB105_5:
-; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
-; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
-; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
-; GFX11-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
-; GFX11-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
-; GFX11-NEXT: v_mov_b32_e32 v13, s12
-; GFX11-NEXT: v_mov_b32_e32 v15, s9
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB105_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v18
-; GFX11-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-NEXT: v_mov_b32_e32 v8, v16
-; GFX11-NEXT: v_mov_b32_e32 v12, v17
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8f16_to_v16i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB105_3
+; GFX11-TRUE16-NEXT: .LBB105_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB105_6
+; GFX11-TRUE16-NEXT: .LBB105_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8f16_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB105_3
+; GFX11-FAKE16-NEXT: .LBB105_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v19, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v17, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v16, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v18, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-FAKE16-NEXT: s_branch .LBB105_6
+; GFX11-FAKE16-NEXT: .LBB105_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s17 :: v_dual_mov_b32 v6, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v10, s11
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v14, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, s12
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s9
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB105_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v19
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v17
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -24355,10 +24820,10 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -24371,8 +24836,8 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -24382,10 +24847,10 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -24403,11 +24868,11 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -25352,8 +25817,8 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -25363,7 +25828,7 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
@@ -25371,27 +25836,27 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v16.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
@@ -25400,10 +25865,9 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v19
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
@@ -25418,13 +25882,14 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v9, v7 :: v_dual_and_b32 v9, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
@@ -25432,54 +25897,57 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v6.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v1, v1, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v14.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v14.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8:
@@ -26090,113 +26558,111 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v10, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s3, 16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x7fff, v11
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v17.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v10, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x7fff, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v14.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[17:18]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v18
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
; GFX11-TRUE16-NEXT: .LBB109_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v9, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s3 :: v_dual_mov_b32 v15, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v13, s12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v7, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v5, s10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v13, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v9, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v15, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v1, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v7, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v3, s4
; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v17
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8_scalar:
@@ -26700,10 +27166,10 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -26716,8 +27182,8 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -26727,10 +27193,10 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -26748,11 +27214,11 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index c7cd6ebeca16e..e34ffdf0fdff9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -50,23 +50,15 @@ define half @bitcast_i16_to_f16(i16 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i16_to_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v0.l, 3
-; GFX11-TRUE16-NEXT: .LBB0_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, 3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_f16:
@@ -164,26 +156,47 @@ define inreg half @bitcast_i16_to_f16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_i16_to_f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
-; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_mov_b32 s1, -1
-; GFX11-NEXT: .LBB1_2: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
-; GFX11-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_4: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_i16_to_f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB1_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: .LBB1_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_i16_to_f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB1_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: .LBB1_4: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -258,23 +271,15 @@ define i16 @bitcast_f16_to_i16(half %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f16_to_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB2_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 0x200, v0.l
-; GFX11-TRUE16-NEXT: .LBB2_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, 0x200, v0.l
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_i16:
@@ -398,7 +403,7 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB3_4:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_i16_scalar:
@@ -485,23 +490,15 @@ define bfloat @bitcast_i16_to_bf16(i16 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i16_to_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v0.l, 3
-; GFX11-TRUE16-NEXT: .LBB4_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, 3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_bf16:
@@ -603,26 +600,47 @@ define inreg bfloat @bitcast_i16_to_bf16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_i16_to_bf16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
-; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_mov_b32 s1, -1
-; GFX11-NEXT: .LBB5_2: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
-; GFX11-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_4: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_i16_to_bf16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB5_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: .LBB5_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_i16_to_bf16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB5_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: .LBB5_4: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -711,64 +729,30 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) #0 {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_bf16_to_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_4
-; GFX11-TRUE16-NEXT: s_branch .LBB6_3
-; GFX11-TRUE16-NEXT: .LBB6_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_4
-; GFX11-TRUE16-NEXT: .LBB6_3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: .LBB6_4: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_bf16_to_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB6_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: .LBB6_2: ; %end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_bf16_to_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v1
+; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT: s_cbranch_execz .LBB6_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: .LBB6_2: ; %end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -875,37 +859,69 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_bf16_to_i16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
-; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_mov_b32 s1, -1
-; GFX11-NEXT: .LBB7_2: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
-; GFX11-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_bf16_to_i16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB7_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB7_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_bf16_to_i16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB7_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, v1, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: .LBB7_4:
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -977,23 +993,15 @@ define bfloat @bitcast_f16_to_bf16(half %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f16_to_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB8_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 0x200, v0.l
-; GFX11-TRUE16-NEXT: .LBB8_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, 0x200, v0.l
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_bf16:
@@ -1121,7 +1129,7 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB9_4:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_bf16_scalar:
@@ -1232,64 +1240,30 @@ define half @bitcast_bf16_to_f16(bfloat %a, i32 %b) #0 {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_bf16_to_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
-; GFX11-TRUE16-NEXT: s_branch .LBB10_3
-; GFX11-TRUE16-NEXT: .LBB10_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
-; GFX11-TRUE16-NEXT: .LBB10_3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_bf16_to_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB10_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: .LBB10_2: ; %end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_bf16_to_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v1
+; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT: s_cbranch_execz .LBB10_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: .LBB10_2: ; %end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1396,37 +1370,69 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_bf16_to_f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
-; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_mov_b32 s1, -1
-; GFX11-NEXT: .LBB11_2: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
-; GFX11-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_4:
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_bf16_to_f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB11_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB11_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_bf16_to_f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB11_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, v1, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: .LBB11_4:
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 24cebc1097205..fe8e47d2a5a19 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -5551,32 +5551,32 @@ define <32 x i8> @bitcast_v8i32_to_v32i8(<8 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i32_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -5585,76 +5585,76 @@ define <32 x i8> @bitcast_v8i32_to_v32i8(<8 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB24_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 3, v28
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 3, v27
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v20
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 3, v39
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 3, v37
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 3, v33
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 3, v32
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 3, v34
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 3, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i32_to_v32i8:
@@ -6176,122 +6176,254 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v31, s14
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i32_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s46, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s12, s19, 24
-; GFX11-NEXT: s_lshr_b32 s13, s19, 16
-; GFX11-NEXT: s_lshr_b32 s14, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s20, s18, 8
-; GFX11-NEXT: s_lshr_b32 s21, s17, 24
-; GFX11-NEXT: s_lshr_b32 s22, s17, 16
-; GFX11-NEXT: s_lshr_b32 s23, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s25, s16, 8
-; GFX11-NEXT: s_lshr_b32 s26, s3, 24
-; GFX11-NEXT: s_lshr_b32 s27, s3, 16
-; GFX11-NEXT: s_lshr_b32 s28, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s40, s2, 8
-; GFX11-NEXT: s_lshr_b32 s41, s1, 24
-; GFX11-NEXT: s_lshr_b32 s42, s1, 16
-; GFX11-NEXT: s_lshr_b32 s43, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB25_3
-; GFX11-NEXT: .LBB25_2:
-; GFX11-NEXT: s_mov_b32 s46, -1
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: .LBB25_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s46, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s3, s3, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s17, s17, 3
-; GFX11-NEXT: s_add_i32 s19, s19, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s12, s19, 24
-; GFX11-NEXT: s_lshr_b32 s13, s19, 16
-; GFX11-NEXT: s_lshr_b32 s14, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s20, s18, 8
-; GFX11-NEXT: s_lshr_b32 s21, s17, 24
-; GFX11-NEXT: s_lshr_b32 s22, s17, 16
-; GFX11-NEXT: s_lshr_b32 s23, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s25, s16, 8
-; GFX11-NEXT: s_lshr_b32 s26, s3, 24
-; GFX11-NEXT: s_lshr_b32 s27, s3, 16
-; GFX11-NEXT: s_lshr_b32 s28, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s40, s2, 8
-; GFX11-NEXT: s_lshr_b32 s41, s1, 24
-; GFX11-NEXT: s_lshr_b32 s42, s1, 16
-; GFX11-NEXT: s_lshr_b32 s43, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: .LBB25_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
-; GFX11-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s43
-; GFX11-NEXT: v_dual_mov_b32 v6, s42 :: v_dual_mov_b32 v7, s41
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s40
-; GFX11-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v11, s8
-; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s28
-; GFX11-NEXT: v_dual_mov_b32 v14, s27 :: v_dual_mov_b32 v15, s26
-; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s25
-; GFX11-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v19, s6
-; GFX11-NEXT: v_dual_mov_b32 v20, s17 :: v_dual_mov_b32 v21, s23
-; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s21
-; GFX11-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v25, s20
-; GFX11-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v27, s4
-; GFX11-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
-; GFX11-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i32_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB25_3
+; GFX11-TRUE16-NEXT: .LBB25_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s3, s3, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s17, s17, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s19, s19, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s45
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s44
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s43
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s41
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s40
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s29
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s26
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, s25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, s24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, s23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, s22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, s21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, s18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, s20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, s19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, s12
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i32_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB25_3
+; GFX11-FAKE16-NEXT: .LBB25_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s3, s3, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s17, s17, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s19, s19, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-FAKE16-NEXT: .LBB25_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s42 :: v_dual_mov_b32 v7, s41
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s40
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v11, s8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s28
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v14, s27 :: v_dual_mov_b32 v15, s26
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s25
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v19, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v20, s17 :: v_dual_mov_b32 v21, s23
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s21
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v25, s20
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v27, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6873,22 +7005,22 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v8i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -6899,11 +7031,11 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -6918,21 +7050,21 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -6955,26 +7087,26 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -7024,7 +7156,7 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -13042,32 +13174,32 @@ define <32 x i8> @bitcast_v8f32_to_v32i8(<8 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8f32_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -13076,73 +13208,74 @@ define <32 x i8> @bitcast_v8f32_to_v32i8(<8 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v27, 1.0, v27
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v28, 1.0, v28 :: v_dual_add_f32 v3, 1.0, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 1.0, v12 :: v_dual_add_f32 v19, 1.0, v19
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 1.0, v20 :: v_dual_add_f32 v11, 1.0, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v39, 1.0, v39 :: v_dual_add_f32 v32, 1.0, v32
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 1.0, v37 :: v_dual_add_f32 v34, 1.0, v34
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 1.0, v35 :: v_dual_add_f32 v36, 1.0, v36
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 1.0, v33 :: v_dual_add_f32 v38, 1.0, v38
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f32_to_v32i8:
@@ -13696,135 +13829,266 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v25, v32
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8f32_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s13, s19, 24
-; GFX11-NEXT: s_lshr_b32 s14, s19, 16
-; GFX11-NEXT: s_lshr_b32 s20, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s21, s18, 8
-; GFX11-NEXT: s_lshr_b32 s22, s17, 24
-; GFX11-NEXT: s_lshr_b32 s23, s17, 16
-; GFX11-NEXT: s_lshr_b32 s25, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s26, s16, 8
-; GFX11-NEXT: s_lshr_b32 s27, s3, 24
-; GFX11-NEXT: s_lshr_b32 s28, s3, 16
-; GFX11-NEXT: s_lshr_b32 s40, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s41, s2, 8
-; GFX11-NEXT: s_lshr_b32 s42, s1, 24
-; GFX11-NEXT: s_lshr_b32 s43, s1, 16
-; GFX11-NEXT: s_lshr_b32 s45, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s46, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB49_3
-; GFX11-NEXT: .LBB49_2:
-; GFX11-NEXT: s_mov_b32 s12, -1
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: .LBB49_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v39, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v37, s3, 1.0
-; GFX11-NEXT: v_add_f32_e64 v35, s17, 1.0
-; GFX11-NEXT: v_add_f32_e64 v33, s19, 1.0
-; GFX11-NEXT: v_add_f32_e64 v32, s18, 1.0
-; GFX11-NEXT: v_add_f32_e64 v34, s16, 1.0
-; GFX11-NEXT: v_add_f32_e64 v36, s2, 1.0
-; GFX11-NEXT: v_add_f32_e64 v38, s0, 1.0
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB49_6
-; GFX11-NEXT: .LBB49_5:
-; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
-; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
-; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
-; GFX11-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
-; GFX11-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
-; GFX11-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
-; GFX11-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
-; GFX11-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
-; GFX11-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
-; GFX11-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
-; GFX11-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
-; GFX11-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
-; GFX11-NEXT: v_mov_b32_e32 v23, s22
-; GFX11-NEXT: v_mov_b32_e32 v25, s21
-; GFX11-NEXT: v_mov_b32_e32 v29, s20
-; GFX11-NEXT: v_mov_b32_e32 v31, s13
-; GFX11-NEXT: v_mov_b32_e32 v27, s10
-; GFX11-NEXT: v_mov_b32_e32 v19, s8
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB49_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v38
-; GFX11-NEXT: v_mov_b32_e32 v4, v39
-; GFX11-NEXT: v_mov_b32_e32 v8, v36
-; GFX11-NEXT: v_mov_b32_e32 v12, v37
-; GFX11-NEXT: v_mov_b32_e32 v16, v34
-; GFX11-NEXT: v_mov_b32_e32 v20, v35
-; GFX11-NEXT: v_mov_b32_e32 v24, v32
-; GFX11-NEXT: v_mov_b32_e32 v28, v33
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8f32_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, s3, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v20, s17, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v28, s19, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v27, s18, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s16, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, s2, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s16 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s18 :: v_dual_mov_b32 v28, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s45 :: v_dual_mov_b32 v7, s42
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s43 :: v_dual_mov_b32 v9, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v15, s27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s40 :: v_dual_mov_b32 v17, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s28 :: v_dual_mov_b32 v23, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v25, s21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s25 :: v_dual_mov_b32 v31, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s23 :: v_dual_mov_b32 v29, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v21, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s20 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s14 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v32.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8f32_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v39, s1, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v37, s3, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v35, s17, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v33, s19, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v32, s18, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v34, s16, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v36, s2, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v38, s0, 1.0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v23, s22
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v25, s21
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s20
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v31, s13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v27, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, s8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v38
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v39
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v36
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v37
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v16, v34
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v32
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v33
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14406,22 +14670,22 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v8f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -14432,11 +14696,11 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -14451,21 +14715,21 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -14488,26 +14752,26 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -14557,7 +14821,7 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -20031,32 +20295,32 @@ define <32 x i8> @bitcast_v4i64_to_v32i8(<4 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4i64_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -20065,79 +20329,79 @@ define <32 x i8> @bitcast_v4i64_to_v32i8(<4 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_co_u32 v11, vcc_lo, v11, 3
+; GFX11-TRUE16-NEXT: v_add_co_u32 v36, vcc_lo, v36, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v19, vcc_lo, v19, 3
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v20, null, 0, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v27, vcc_lo, v27, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v37, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v34, vcc_lo, v34, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v35, null, 0, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v32, vcc_lo, v32, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v28, null, 0, v28, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v3, 3
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v38, vcc_lo, v38, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v39, null, 0, v39, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i64_to_v32i8:
@@ -20662,122 +20926,254 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v31, s14
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i64_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s46, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s12, s19, 24
-; GFX11-NEXT: s_lshr_b32 s13, s19, 16
-; GFX11-NEXT: s_lshr_b32 s14, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s20, s18, 8
-; GFX11-NEXT: s_lshr_b32 s21, s17, 24
-; GFX11-NEXT: s_lshr_b32 s22, s17, 16
-; GFX11-NEXT: s_lshr_b32 s23, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s25, s16, 8
-; GFX11-NEXT: s_lshr_b32 s26, s3, 24
-; GFX11-NEXT: s_lshr_b32 s27, s3, 16
-; GFX11-NEXT: s_lshr_b32 s28, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s40, s2, 8
-; GFX11-NEXT: s_lshr_b32 s41, s1, 24
-; GFX11-NEXT: s_lshr_b32 s42, s1, 16
-; GFX11-NEXT: s_lshr_b32 s43, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB69_3
-; GFX11-NEXT: .LBB69_2:
-; GFX11-NEXT: s_mov_b32 s46, -1
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: .LBB69_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s46, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_u32 s0, s0, 3
-; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: s_add_u32 s2, s2, 3
-; GFX11-NEXT: s_addc_u32 s3, s3, 0
-; GFX11-NEXT: s_add_u32 s16, s16, 3
-; GFX11-NEXT: s_addc_u32 s17, s17, 0
-; GFX11-NEXT: s_add_u32 s18, s18, 3
-; GFX11-NEXT: s_addc_u32 s19, s19, 0
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s12, s19, 24
-; GFX11-NEXT: s_lshr_b32 s13, s19, 16
-; GFX11-NEXT: s_lshr_b32 s14, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s20, s18, 8
-; GFX11-NEXT: s_lshr_b32 s21, s17, 24
-; GFX11-NEXT: s_lshr_b32 s22, s17, 16
-; GFX11-NEXT: s_lshr_b32 s23, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s25, s16, 8
-; GFX11-NEXT: s_lshr_b32 s26, s3, 24
-; GFX11-NEXT: s_lshr_b32 s27, s3, 16
-; GFX11-NEXT: s_lshr_b32 s28, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s40, s2, 8
-; GFX11-NEXT: s_lshr_b32 s41, s1, 24
-; GFX11-NEXT: s_lshr_b32 s42, s1, 16
-; GFX11-NEXT: s_lshr_b32 s43, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: .LBB69_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
-; GFX11-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s43
-; GFX11-NEXT: v_dual_mov_b32 v6, s42 :: v_dual_mov_b32 v7, s41
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s40
-; GFX11-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v11, s8
-; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s28
-; GFX11-NEXT: v_dual_mov_b32 v14, s27 :: v_dual_mov_b32 v15, s26
-; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s25
-; GFX11-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v19, s6
-; GFX11-NEXT: v_dual_mov_b32 v20, s17 :: v_dual_mov_b32 v21, s23
-; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s21
-; GFX11-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v25, s20
-; GFX11-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v27, s4
-; GFX11-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
-; GFX11-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i64_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB69_3
+; GFX11-TRUE16-NEXT: .LBB69_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-TRUE16-NEXT: s_add_u32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, 0
+; GFX11-TRUE16-NEXT: s_add_u32 s16, s16, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s17, s17, 0
+; GFX11-TRUE16-NEXT: s_add_u32 s18, s18, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s19, s19, 0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s45
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s44
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s43
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s41
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s40
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s29
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s26
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, s25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, s24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, s23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, s22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, s21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, s18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, s20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, s19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, s12
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i64_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB69_3
+; GFX11-FAKE16-NEXT: .LBB69_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-FAKE16-NEXT: s_add_u32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s3, s3, 0
+; GFX11-FAKE16-NEXT: s_add_u32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s17, s17, 0
+; GFX11-FAKE16-NEXT: s_add_u32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s19, s19, 0
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-FAKE16-NEXT: .LBB69_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s42 :: v_dual_mov_b32 v7, s41
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s40
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v11, s8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s28
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v14, s27 :: v_dual_mov_b32 v15, s26
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s25
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v19, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v20, s17 :: v_dual_mov_b32 v21, s23
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s21
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v25, s20
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v27, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -21359,22 +21755,22 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -21385,11 +21781,11 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -21404,21 +21800,21 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -21441,26 +21837,26 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -21510,7 +21906,7 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -26452,32 +26848,32 @@ define <32 x i8> @bitcast_v4f64_to_v32i8(<4 x double> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4f64_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -26486,74 +26882,74 @@ define <32 x i8> @bitcast_v4f64_to_v32i8(<4 x double> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB84_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[27:28], v[27:28], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[19:20], v[19:20], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], v[11:12], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], v[3:4], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[32:33], v[32:33], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[34:35], v[34:35], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[36:37], v[36:37], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[38:39], v[38:39], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f64_to_v32i8:
@@ -27095,133 +27491,262 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v25, v32
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4f64_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s28, s19, 24
-; GFX11-NEXT: s_lshr_b32 s27, s19, 16
-; GFX11-NEXT: s_lshr_b32 s26, s19, 8
-; GFX11-NEXT: s_lshr_b32 s40, s18, 16
-; GFX11-NEXT: s_lshr_b32 s29, s18, 8
-; GFX11-NEXT: s_lshr_b32 s25, s17, 24
-; GFX11-NEXT: s_lshr_b32 s24, s17, 16
-; GFX11-NEXT: s_lshr_b32 s23, s17, 8
-; GFX11-NEXT: s_lshr_b32 s42, s16, 16
-; GFX11-NEXT: s_lshr_b32 s41, s16, 8
-; GFX11-NEXT: s_lshr_b32 s22, s3, 24
-; GFX11-NEXT: s_lshr_b32 s21, s3, 16
-; GFX11-NEXT: s_lshr_b32 s20, s3, 8
-; GFX11-NEXT: s_lshr_b32 s44, s2, 16
-; GFX11-NEXT: s_lshr_b32 s43, s2, 8
-; GFX11-NEXT: s_lshr_b32 s15, s1, 24
-; GFX11-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB85_3
-; GFX11-NEXT: .LBB85_2:
-; GFX11-NEXT: s_mov_b32 s12, -1
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: .LBB85_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f64 v[32:33], s[18:19], 1.0
-; GFX11-NEXT: v_add_f64 v[34:35], s[16:17], 1.0
-; GFX11-NEXT: v_add_f64 v[36:37], s[2:3], 1.0
-; GFX11-NEXT: v_add_f64 v[38:39], s[0:1], 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB85_6
-; GFX11-NEXT: .LBB85_5:
-; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v33, s19
-; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v35, s17
-; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v37, s3
-; GFX11-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v39, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s46 :: v_dual_mov_b32 v1, s45
-; GFX11-NEXT: v_dual_mov_b32 v10, s44 :: v_dual_mov_b32 v9, s43
-; GFX11-NEXT: v_dual_mov_b32 v18, s42 :: v_dual_mov_b32 v17, s41
-; GFX11-NEXT: v_dual_mov_b32 v26, s40 :: v_dual_mov_b32 v25, s29
-; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v30, s27
-; GFX11-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v22, s24
-; GFX11-NEXT: v_dual_mov_b32 v19, s8 :: v_dual_mov_b32 v14, s21
-; GFX11-NEXT: v_dual_mov_b32 v27, s10 :: v_dual_mov_b32 v6, s14
-; GFX11-NEXT: v_mov_b32_e32 v31, s28
-; GFX11-NEXT: v_mov_b32_e32 v29, s26
-; GFX11-NEXT: v_mov_b32_e32 v23, s25
-; GFX11-NEXT: v_mov_b32_e32 v21, s23
-; GFX11-NEXT: v_mov_b32_e32 v15, s22
-; GFX11-NEXT: v_mov_b32_e32 v13, s20
-; GFX11-NEXT: v_mov_b32_e32 v7, s15
-; GFX11-NEXT: v_mov_b32_e32 v5, s13
-; GFX11-NEXT: .LBB85_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v38
-; GFX11-NEXT: v_mov_b32_e32 v4, v39
-; GFX11-NEXT: v_mov_b32_e32 v8, v36
-; GFX11-NEXT: v_mov_b32_e32 v12, v37
-; GFX11-NEXT: v_mov_b32_e32 v16, v34
-; GFX11-NEXT: v_mov_b32_e32 v20, v35
-; GFX11-NEXT: v_mov_b32_e32 v24, v32
-; GFX11-NEXT: v_mov_b32_e32 v28, v33
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4f64_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB85_3
+; GFX11-TRUE16-NEXT: .LBB85_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f64 v[27:28], s[18:19], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[19:20], s[16:17], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], s[2:3], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[19:20]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB85_6
+; GFX11-TRUE16-NEXT: .LBB85_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v28, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s16 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s18 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s46 :: v_dual_mov_b32 v1, s45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s44 :: v_dual_mov_b32 v9, s43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s42 :: v_dual_mov_b32 v17, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s40 :: v_dual_mov_b32 v25, s29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v30, s27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s6 :: v_dual_mov_b32 v32, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v22, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s10 :: v_dual_mov_b32 v24, s23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s28 :: v_dual_mov_b32 v14, s21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v16, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s22 :: v_dual_mov_b32 v6, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s15 :: v_dual_mov_b32 v8, s13
+; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v32.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4f64_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB85_3
+; GFX11-FAKE16-NEXT: .LBB85_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f64 v[32:33], s[18:19], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[34:35], s[16:17], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[36:37], s[2:3], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[38:39], s[0:1], 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-FAKE16-NEXT: s_branch .LBB85_6
+; GFX11-FAKE16-NEXT: .LBB85_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v33, s19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v35, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v37, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v39, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s46 :: v_dual_mov_b32 v1, s45
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s44 :: v_dual_mov_b32 v9, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s42 :: v_dual_mov_b32 v17, s41
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s40 :: v_dual_mov_b32 v25, s29
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v30, s27
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v22, s24
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s8 :: v_dual_mov_b32 v14, s21
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s10 :: v_dual_mov_b32 v6, s14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v31, s28
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s26
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v23, s25
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v21, s23
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s22
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, s20
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s15
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s13
+; GFX11-FAKE16-NEXT: .LBB85_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v38
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v39
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v36
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v37
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v16, v34
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v32
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v33
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -27803,22 +28328,22 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -27829,11 +28354,11 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -27848,21 +28373,21 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -27885,26 +28410,26 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -27954,7 +28479,7 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -32563,32 +33088,32 @@ define <32 x i8> @bitcast_v16i16_to_v32i8(<16 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i16_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -32597,76 +33122,76 @@ define <32 x i8> @bitcast_v16i16_to_v32i8(<16 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v39, v39, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v37, v37, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v35, v35, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v33, v33, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v32, v32, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v34, v34, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v36, v36, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v38, v38, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16i16_to_v32i8:
@@ -33290,135 +33815,266 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v25, v32
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i16_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s13, s19, 24
-; GFX11-NEXT: s_lshr_b32 s14, s19, 16
-; GFX11-NEXT: s_lshr_b32 s20, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s21, s18, 8
-; GFX11-NEXT: s_lshr_b32 s22, s17, 24
-; GFX11-NEXT: s_lshr_b32 s23, s17, 16
-; GFX11-NEXT: s_lshr_b32 s25, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s26, s16, 8
-; GFX11-NEXT: s_lshr_b32 s27, s3, 24
-; GFX11-NEXT: s_lshr_b32 s28, s3, 16
-; GFX11-NEXT: s_lshr_b32 s40, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s41, s2, 8
-; GFX11-NEXT: s_lshr_b32 s42, s1, 24
-; GFX11-NEXT: s_lshr_b32 s43, s1, 16
-; GFX11-NEXT: s_lshr_b32 s45, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s46, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB97_3
-; GFX11-NEXT: .LBB97_2:
-; GFX11-NEXT: s_mov_b32 s12, -1
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: .LBB97_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v39, s1, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v37, s3, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v35, s17, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v33, s19, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v32, s18, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v34, s16, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v36, s2, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v38, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB97_6
-; GFX11-NEXT: .LBB97_5:
-; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
-; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
-; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
-; GFX11-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
-; GFX11-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
-; GFX11-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
-; GFX11-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
-; GFX11-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
-; GFX11-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
-; GFX11-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
-; GFX11-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
-; GFX11-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
-; GFX11-NEXT: v_mov_b32_e32 v23, s22
-; GFX11-NEXT: v_mov_b32_e32 v25, s21
-; GFX11-NEXT: v_mov_b32_e32 v29, s20
-; GFX11-NEXT: v_mov_b32_e32 v31, s13
-; GFX11-NEXT: v_mov_b32_e32 v27, s10
-; GFX11-NEXT: v_mov_b32_e32 v19, s8
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB97_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v38
-; GFX11-NEXT: v_mov_b32_e32 v4, v39
-; GFX11-NEXT: v_mov_b32_e32 v8, v36
-; GFX11-NEXT: v_mov_b32_e32 v12, v37
-; GFX11-NEXT: v_mov_b32_e32 v16, v34
-; GFX11-NEXT: v_mov_b32_e32 v20, v35
-; GFX11-NEXT: v_mov_b32_e32 v24, v32
-; GFX11-NEXT: v_mov_b32_e32 v28, v33
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i16_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB97_3
+; GFX11-TRUE16-NEXT: .LBB97_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, s3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v20, s17, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v28, s19, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v27, s18, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, s16, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, s2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB97_6
+; GFX11-TRUE16-NEXT: .LBB97_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s16 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s18 :: v_dual_mov_b32 v28, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s45 :: v_dual_mov_b32 v7, s42
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s43 :: v_dual_mov_b32 v9, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v15, s27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s40 :: v_dual_mov_b32 v17, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s28 :: v_dual_mov_b32 v23, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v25, s21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s25 :: v_dual_mov_b32 v31, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s23 :: v_dual_mov_b32 v29, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v21, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s20 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s14 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v32.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i16_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB97_3
+; GFX11-FAKE16-NEXT: .LBB97_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v39, s1, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v37, s3, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v35, s17, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v33, s19, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v32, s18, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v34, s16, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v36, s2, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v38, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-FAKE16-NEXT: s_branch .LBB97_6
+; GFX11-FAKE16-NEXT: .LBB97_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v23, s22
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v25, s21
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s20
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v31, s13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v27, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, s8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB97_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v38
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v39
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v36
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v37
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v16, v34
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v32
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v33
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -34047,44 +34703,29 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB98_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -34094,133 +34735,133 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -35052,19 +35693,19 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v14
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v9
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
@@ -35072,23 +35713,23 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s44, s29, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s12, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s13, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s8, 8
; GFX11-TRUE16-NEXT: s_or_b32 s43, s43, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s18, s19, v4
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s43, s43, s44
-; GFX11-TRUE16-NEXT: s_and_b32 s44, s11, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s44, s10, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s45, s7, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s2, s3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s44, s44, s45
; GFX11-TRUE16-NEXT: s_and_b32 s45, s40, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s10, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s8, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s11, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s4, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s45, s45, s46
@@ -35104,8 +35745,8 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s44, s44, s45
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s15, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s13, s6, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s12, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
@@ -35122,33 +35763,33 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s13, s6, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s15, s6, v0
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s8, s4, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s11, s4, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s14, s5, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s15, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s12, v0
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s12, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s28, s29, v0
; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s20, s21, v0
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s11, s7, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s10, s7, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s26, s27, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v1
@@ -38094,32 +38735,32 @@ define <32 x i8> @bitcast_v16f16_to_v32i8(<16 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16f16_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -38128,76 +38769,76 @@ define <32 x i8> @bitcast_v16f16_to_v32i8(<16 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB104_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v39, 0x200, v39 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v37, 0x200, v37 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v35, 0x200, v35 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v33, 0x200, v33 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v32, 0x200, v32 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v34, 0x200, v34 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v36, 0x200, v36 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v38, 0x200, v38 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16f16_to_v32i8:
@@ -38855,135 +39496,266 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v25, v32
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16f16_to_v32i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s13, s19, 24
-; GFX11-NEXT: s_lshr_b32 s14, s19, 16
-; GFX11-NEXT: s_lshr_b32 s20, s19, 8
-; GFX11-NEXT: s_lshr_b32 s15, s18, 16
-; GFX11-NEXT: s_lshr_b32 s21, s18, 8
-; GFX11-NEXT: s_lshr_b32 s22, s17, 24
-; GFX11-NEXT: s_lshr_b32 s23, s17, 16
-; GFX11-NEXT: s_lshr_b32 s25, s17, 8
-; GFX11-NEXT: s_lshr_b32 s24, s16, 16
-; GFX11-NEXT: s_lshr_b32 s26, s16, 8
-; GFX11-NEXT: s_lshr_b32 s27, s3, 24
-; GFX11-NEXT: s_lshr_b32 s28, s3, 16
-; GFX11-NEXT: s_lshr_b32 s40, s3, 8
-; GFX11-NEXT: s_lshr_b32 s29, s2, 16
-; GFX11-NEXT: s_lshr_b32 s41, s2, 8
-; GFX11-NEXT: s_lshr_b32 s42, s1, 24
-; GFX11-NEXT: s_lshr_b32 s43, s1, 16
-; GFX11-NEXT: s_lshr_b32 s45, s1, 8
-; GFX11-NEXT: s_lshr_b32 s44, s0, 16
-; GFX11-NEXT: s_lshr_b32 s46, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB105_3
-; GFX11-NEXT: .LBB105_2:
-; GFX11-NEXT: s_mov_b32 s12, -1
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: .LBB105_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_f16 v39, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v37, 0x200, s3 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v35, 0x200, s17 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v33, 0x200, s19 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v32, 0x200, s18 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v34, 0x200, s16 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v36, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v38, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB105_6
-; GFX11-NEXT: .LBB105_5:
-; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
-; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
-; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
-; GFX11-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
-; GFX11-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
-; GFX11-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
-; GFX11-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
-; GFX11-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
-; GFX11-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
-; GFX11-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
-; GFX11-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
-; GFX11-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
-; GFX11-NEXT: v_mov_b32_e32 v23, s22
-; GFX11-NEXT: v_mov_b32_e32 v25, s21
-; GFX11-NEXT: v_mov_b32_e32 v29, s20
-; GFX11-NEXT: v_mov_b32_e32 v31, s13
-; GFX11-NEXT: v_mov_b32_e32 v27, s10
-; GFX11-NEXT: v_mov_b32_e32 v19, s8
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB105_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v38
-; GFX11-NEXT: v_mov_b32_e32 v4, v39
-; GFX11-NEXT: v_mov_b32_e32 v8, v36
-; GFX11-NEXT: v_mov_b32_e32 v12, v37
-; GFX11-NEXT: v_mov_b32_e32 v16, v34
-; GFX11-NEXT: v_mov_b32_e32 v20, v35
-; GFX11-NEXT: v_mov_b32_e32 v24, v32
-; GFX11-NEXT: v_mov_b32_e32 v28, v33
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16f16_to_v32i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB105_3
+; GFX11-TRUE16-NEXT: .LBB105_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v20, 0x200, s17 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v28, 0x200, s19 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v27, 0x200, s18 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, s16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB105_6
+; GFX11-TRUE16-NEXT: .LBB105_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s16 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s18 :: v_dual_mov_b32 v28, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s45 :: v_dual_mov_b32 v7, s42
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s43 :: v_dual_mov_b32 v9, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s29 :: v_dual_mov_b32 v15, s27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s40 :: v_dual_mov_b32 v17, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s28 :: v_dual_mov_b32 v23, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s24 :: v_dual_mov_b32 v25, s21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s25 :: v_dual_mov_b32 v31, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s23 :: v_dual_mov_b32 v29, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s15 :: v_dual_mov_b32 v21, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s20 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s14 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v32.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16f16_to_v32i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s19, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s20, s19, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s21, s18, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s22, s17, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s23, s17, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s25, s17, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s26, s16, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s27, s3, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s28, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s3, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s29, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB105_3
+; GFX11-FAKE16-NEXT: .LBB105_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v39, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v37, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v35, 0x200, s17 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v33, 0x200, s19 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v32, 0x200, s18 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v34, 0x200, s16 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v36, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v38, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-FAKE16-NEXT: s_branch .LBB105_6
+; GFX11-FAKE16-NEXT: .LBB105_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s18 :: v_dual_mov_b32 v33, s19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s44
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v6, s43
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s42 :: v_dual_mov_b32 v10, s29
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s41 :: v_dual_mov_b32 v14, s28
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v18, s24
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v22, s23
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v26, s15
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v30, s14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v23, s22
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v25, s21
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s20
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v31, s13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v27, s10
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, s8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB105_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v38
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v39
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v36
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v37
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v16, v34
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v32
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v33
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -39612,44 +40384,29 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB106_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -39659,133 +40416,133 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -40617,19 +41374,19 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v14
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v9
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
@@ -40637,23 +41394,23 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s44, s29, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s12, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s13, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s8, 8
; GFX11-TRUE16-NEXT: s_or_b32 s43, s43, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s18, s19, v4
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s43, s43, s44
-; GFX11-TRUE16-NEXT: s_and_b32 s44, s11, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s44, s10, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s45, s7, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s2, s3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s44, s44, s45
; GFX11-TRUE16-NEXT: s_and_b32 s45, s40, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s10, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s8, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s11, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s4, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s45, s45, s46
@@ -40669,8 +41426,8 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s44, s44, s45
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s15, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s13, s6, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s12, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
@@ -40687,33 +41444,33 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s13, s6, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s15, s6, v0
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s8, s4, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s11, s4, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s14, s5, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s15, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s12, v0
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s12, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s28, s29, v0
; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s20, s21, v0
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s11, s7, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s10, s7, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s26, s27, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v1
@@ -41596,32 +42353,32 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v7 :: v_dual_mov_b32 v27, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -41630,210 +42387,213 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v4, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v39
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v39
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v5, v7 :: v_dual_lshlrev_b32 v3, 16, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v37
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v7 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v5 :: v_dual_and_b32 v9, 0xffff0000, v34
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v36
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v8, v10 :: v_dual_and_b32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v36
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v5, v8 :: v_dual_and_b32 v5, 0xffff0000, v35
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v14.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v14.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v35
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v8 :: v_dual_add_f32 v5, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v10 :: v_dual_lshlrev_b32 v0, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v34
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v22.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v13 :: v_dual_lshlrev_b32 v8, 16, v28
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v22.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v33
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v7, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v18.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v11, v8, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v30.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v26.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v30.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v26.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v32i8:
@@ -42942,204 +43702,209 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, 0xffff0000
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, 0xffff0000
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, 0xffff0000
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s3
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, 0xffff0000
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v10, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v9, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v10, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v5 :: v_dual_add_nc_u32 v4, 0x7fff, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s18, 16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v35.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v15, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_add_nc_u32 v3, v3, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x7fff, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v10, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, v16, v15
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x7fff, v16
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v7, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s17, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v34.l
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v13.l
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: s_and_b32 s0, s16, 0xffff0000
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v7 :: v_dual_add_nc_u32 v4, v4, v9
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v9 :: v_dual_add_nc_u32 v4, v4, v11
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s16, 16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s19, 0xffff0000
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v5 :: v_dual_add_nc_u32 v7, v11, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v12, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v4
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s19, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v11, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v4
; GFX11-TRUE16-NEXT: s_and_b32 s0, s18, 0xffff0000
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v33.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v21.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v15
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v22.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v18.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v32.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v26.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[50:51]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v50
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v36
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v29.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[35:36]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[33:34]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v24.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[37:38]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v38
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v26.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[48:49]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v48
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v33
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
; GFX11-TRUE16-NEXT: .LBB109_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v33, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s46 :: v_dual_mov_b32 v35, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s19 :: v_dual_mov_b32 v25, s28
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s45 :: v_dual_mov_b32 v31, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v29, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s44 :: v_dual_mov_b32 v17, s27
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s43 :: v_dual_mov_b32 v23, s21
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v21, s23
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s42 :: v_dual_mov_b32 v9, s25
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s3 :: v_dual_mov_b32 v15, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s41 :: v_dual_mov_b32 v13, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s40 :: v_dual_mov_b32 v7, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s29 :: v_dual_mov_b32 v5, s14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v29, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s46 :: v_dual_mov_b32 v21, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s45 :: v_dual_mov_b32 v13, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s44 :: v_dual_mov_b32 v25, s28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s43 :: v_dual_mov_b32 v31, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v17, s27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s42 :: v_dual_mov_b32 v23, s21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s41 :: v_dual_mov_b32 v9, s25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v15, s15
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s40 :: v_dual_mov_b32 v1, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s29 :: v_dual_mov_b32 v7, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s26 :: v_dual_mov_b32 v27, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s23 :: v_dual_mov_b32 v19, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v3, s4
; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v35
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v34
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v20, v33
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v28, v32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v32.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v32i8_scalar:
@@ -44047,44 +44812,29 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB110_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -44094,133 +44844,133 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -45044,19 +45794,19 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v14
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v9
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
@@ -45064,23 +45814,23 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s44, s29, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s12, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s13, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s8, 8
; GFX11-TRUE16-NEXT: s_or_b32 s43, s43, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s18, s19, v4
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s43, s43, s44
-; GFX11-TRUE16-NEXT: s_and_b32 s44, s11, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s44, s10, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s45, s7, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s2, s3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s44, s44, s45
; GFX11-TRUE16-NEXT: s_and_b32 s45, s40, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s10, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s9, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
; GFX11-TRUE16-NEXT: s_or_b32 s44, s45, s46
-; GFX11-TRUE16-NEXT: s_and_b32 s45, s8, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s45, s11, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s46, s4, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s45, s45, s46
@@ -45096,8 +45846,8 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s44, s44, s45
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s15, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s13, s6, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s12, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
@@ -45114,33 +45864,33 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s13, s6, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s15, s6, v0
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s8, s4, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s11, s4, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s14, s5, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s15, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s41, s12, v0
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s12, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s28, s29, v0
; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s20, s21, v0
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s11, s7, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s9, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s10, s7, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s26, s27, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index c0fcebcf00814..c41b685ff2d16 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -4903,31 +4903,32 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v10i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -4938,11 +4939,11 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB14_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v51, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v49, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v14, v15, 0xc0c0004
@@ -4959,26 +4960,26 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v64, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v55, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v51, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
@@ -5000,39 +5001,39 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2
; GFX11-TRUE16-NEXT: .LBB14_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v10.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v31.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v12.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v14.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
@@ -5068,18 +5069,18 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v51.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v54.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -5088,12 +5089,12 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -11809,31 +11810,32 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v10f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -11844,11 +11846,11 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB34_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v51, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v49, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v14, v15, 0xc0c0004
@@ -11865,26 +11867,26 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v64, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v55, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v51, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
@@ -11906,39 +11908,39 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_2
; GFX11-TRUE16-NEXT: .LBB34_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v10.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v31.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v12.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v14.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
@@ -11974,18 +11976,18 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v51.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v54.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -11994,12 +11996,12 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -18386,57 +18388,42 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v2, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v1.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -18447,173 +18434,174 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v51, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v49, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v18.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v19.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v48, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v17.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v65, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v20.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v64.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v28.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v17.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
@@ -19665,34 +19653,34 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v17
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v16
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v15
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_and_b32 s59, s43, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s60, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s59, s47, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s60, s13, 8
; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s4, 8
; GFX11-TRUE16-NEXT: s_or_b32 s59, s59, s60
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
@@ -19704,29 +19692,29 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s45, s14, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s60, s6, 0xff
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s11, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s60, s42, 0xff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s42, s13, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s40, s10, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s8, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s12, s7, v8
; GFX11-TRUE16-NEXT: s_or_b32 s60, s60, s61
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s12, s9, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s62, s11, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s7, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s10, s5, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s62, s43, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s6, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s60
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s57, s56, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s57, s40, v8
; GFX11-TRUE16-NEXT: s_or_b32 s60, s62, s61
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s47, s44, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s45, s14, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s46, s41, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s56, s44, v8
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s59, s60
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s15, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s46, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s59
; GFX11-TRUE16-NEXT: s_branch .LBB51_3
@@ -19741,45 +19729,45 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s6, s6, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s15, s5, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s6, s4, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s11, s7, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s46, s9, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s42, s4, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s6, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s46, s41, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s56, s44, v0
; GFX11-TRUE16-NEXT: s_add_i32 s57, s57, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s12, s9, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s47, s44, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s57, s56, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s10, s5, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s45, s14, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s57, s40, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s28, s29, v0
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s43, s8, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s47, s13, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s45, s14, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s41, s11, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s20, s21, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s42, s13, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s12, s7, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s15, s8, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s26, s27, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v1
@@ -24837,57 +24825,42 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v2, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v1.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -24898,173 +24871,174 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB62_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v51, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v49, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v18.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v19.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v48, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v17.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v65, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
; GFX11-TRUE16-NEXT: .LBB62_4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v20.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v64.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v28.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v17.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
@@ -26116,34 +26090,34 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v17
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v16
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v15
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_and_b32 s59, s43, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s60, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s59, s47, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s60, s13, 8
; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s4, 8
; GFX11-TRUE16-NEXT: s_or_b32 s59, s59, s60
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
@@ -26155,29 +26129,29 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s45, s14, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s60, s6, 0xff
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s41, s11, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s60, s42, 0xff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s42, s13, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s40, s10, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s8, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s12, s7, v8
; GFX11-TRUE16-NEXT: s_or_b32 s60, s60, s61
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s12, s9, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s62, s11, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s7, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s10, s5, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s62, s43, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s61, s6, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s60
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s57, s56, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s57, s40, v8
; GFX11-TRUE16-NEXT: s_or_b32 s60, s62, s61
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s47, s44, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s45, s14, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s46, s41, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s56, s44, v8
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s59, s60
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s15, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s46, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s59
; GFX11-TRUE16-NEXT: s_branch .LBB63_3
@@ -26192,45 +26166,45 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s6, s6, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s15, s5, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s6, s4, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s11, s7, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s46, s9, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s42, s4, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s6, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s46, s41, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s56, s44, v0
; GFX11-TRUE16-NEXT: s_add_i32 s57, s57, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, s12, s9, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s47, s44, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s57, s56, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, s10, s5, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s45, s14, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s57, s40, v0
+; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s28, s29, v0
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s43, s8, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s47, s13, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s45, s14, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s41, s11, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s20, s21, v0
-; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s42, s13, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s12, s7, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s15, s8, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s26, s27, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v1
@@ -29328,38 +29302,38 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v5f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB72_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -29370,15 +29344,15 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB72_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v71, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v69, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v67, v66, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v53, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v49, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v48, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v36, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v35, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
@@ -29391,33 +29365,33 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -29432,46 +29406,46 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_2
; GFX11-TRUE16-NEXT: .LBB72_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v53.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v23.l
@@ -29479,7 +29453,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v3.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
@@ -29500,18 +29474,18 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v65.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -29520,12 +29494,12 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -33193,38 +33167,38 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v5i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -33235,15 +33209,15 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB76_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v71, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v69, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v67, v66, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v53, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v49, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v48, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v36, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v35, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
@@ -33256,33 +33230,33 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -33297,46 +33271,46 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2
; GFX11-TRUE16-NEXT: .LBB76_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v53.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v23.l
@@ -33344,7 +33318,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v3.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
@@ -33365,18 +33339,18 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v65.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -33385,12 +33359,12 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 38a3d781676ef..7522d1b697b33 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -2176,22 +2176,25 @@ define <4 x i8> @bitcast_i32_to_v4i8(i32 %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB20_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB20_4
+; GFX11-TRUE16-NEXT: .LBB20_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB20_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB20_3
-; GFX11-TRUE16-NEXT: .LBB20_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: .LBB20_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_2
+; GFX11-TRUE16-NEXT: .LBB20_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2343,39 +2346,74 @@ define inreg <4 x i8> @bitcast_i32_to_v4i8_scalar(i32 inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_i32_to_v4i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_branch .LBB21_3
-; GFX11-NEXT: .LBB21_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: .LBB21_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB21_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: .LBB21_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_i32_to_v4i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB21_3
+; GFX11-TRUE16-NEXT: .LBB21_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: .LBB21_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_i32_to_v4i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB21_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB21_3
+; GFX11-FAKE16-NEXT: .LBB21_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: .LBB21_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-FAKE16-NEXT: .LBB21_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4646,22 +4684,25 @@ define <4 x i8> @bitcast_f32_to_v4i8(float %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_4
+; GFX11-TRUE16-NEXT: .LBB40_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB40_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB40_3
-; GFX11-TRUE16-NEXT: .LBB40_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: .LBB40_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2
+; GFX11-TRUE16-NEXT: .LBB40_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6792,22 +6833,25 @@ define <4 x i8> @bitcast_v2i16_to_v4i8(<2 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB56_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_4
+; GFX11-TRUE16-NEXT: .LBB56_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB56_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB56_3
-; GFX11-TRUE16-NEXT: .LBB56_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: .LBB56_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB56_2
+; GFX11-TRUE16-NEXT: .LBB56_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8574,22 +8618,25 @@ define <4 x i8> @bitcast_v2f16_to_v4i8(<2 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB68_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB68_4
+; GFX11-TRUE16-NEXT: .LBB68_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB68_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB68_3
-; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: .LBB68_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
+; GFX11-TRUE16-NEXT: .LBB68_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -9961,18 +10008,20 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_4
+; GFX11-TRUE16-NEXT: .LBB76_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB76_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_4
-; GFX11-TRUE16-NEXT: s_branch .LBB76_3
-; GFX11-TRUE16-NEXT: .LBB76_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_4
-; GFX11-TRUE16-NEXT: .LBB76_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2
+; GFX11-TRUE16-NEXT: .LBB76_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
@@ -9996,7 +10045,6 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: .LBB76_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -10850,22 +10898,25 @@ define <4 x i8> @bitcast_v1i32_to_v4i8(<1 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB80_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB80_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB80_4
+; GFX11-TRUE16-NEXT: .LBB80_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB80_3: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB80_3
-; GFX11-TRUE16-NEXT: .LBB80_2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: .LBB80_3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB80_2
+; GFX11-TRUE16-NEXT: .LBB80_4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -11017,39 +11068,74 @@ define inreg <4 x i8> @bitcast_v1i32_to_v4i8_scalar(<1 x i32> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v1i32_to_v4i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_branch .LBB81_3
-; GFX11-NEXT: .LBB81_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: .LBB81_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB81_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: .LBB81_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v1i32_to_v4i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB81_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB81_3
+; GFX11-TRUE16-NEXT: .LBB81_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: .LBB81_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: .LBB81_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v1i32_to_v4i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB81_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB81_3
+; GFX11-FAKE16-NEXT: .LBB81_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: .LBB81_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-FAKE16-NEXT: .LBB81_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index fe78e8a8c0764..aa132d78c0afe 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -13114,63 +13114,63 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -13181,15 +13181,15 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v119, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v117, v116, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v115, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v113, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v99, v98, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v97, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v37, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v22, v23, 0xc0c0004
@@ -13203,50 +13203,50 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v86, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v84, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v82, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v71, v69, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v68, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v118, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v116, v115, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v114, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v112, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v103, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v100, v99, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v15, 16, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v65, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v54, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v51, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v48, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v37, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v35, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v98, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v97, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v86, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v85, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v69, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v67, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v16, 16, v15
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v20, 16, v19
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -13261,64 +13261,64 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v115.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v113.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v114.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v112.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v102.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v2.l
@@ -13326,14 +13326,14 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v97.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v26.l, 3
@@ -13353,69 +13353,69 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v24.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v86.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v82.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v114.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v80.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v68.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v112.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v84.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v50.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v83.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v102.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v69.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v99.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v96.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.h, v11.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v85.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v17.l, v15.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v17.h, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h
@@ -13425,14 +13425,14 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v11.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v12.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v12.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
@@ -28309,63 +28309,63 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -28376,15 +28376,15 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v119, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v117, v116, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v115, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v113, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v99, v98, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v97, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v37, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v22, v23, 0xc0c0004
@@ -28398,50 +28398,50 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v86, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v84, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v82, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v71, v69, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v68, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v118, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v116, v115, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v114, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v112, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v103, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v100, v99, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v15, 16, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v65, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v54, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v51, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v48, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v37, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v35, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v98, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v97, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v86, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v85, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v69, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v67, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v16, 16, v15
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v20, 16, v19
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -28456,64 +28456,64 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v115.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v113.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v114.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v112.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v102.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v2.l
@@ -28521,14 +28521,14 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v97.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v26.l, 3
@@ -28548,69 +28548,69 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v24.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v86.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v82.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v114.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v80.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v68.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v112.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v84.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v50.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v83.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v102.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v69.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v99.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v96.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.h, v11.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v85.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v17.l, v15.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v17.h, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h
@@ -28620,14 +28620,14 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v11.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v12.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v12.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
@@ -42382,63 +42382,63 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -42449,15 +42449,15 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v119, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v117, v116, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v115, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v113, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v99, v98, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v97, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v37, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v22, v23, 0xc0c0004
@@ -42471,50 +42471,50 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v86, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v84, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v82, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v71, v69, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v68, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v118, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v116, v115, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v114, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v112, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v103, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v100, v99, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v15, 16, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v65, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v54, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v51, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v48, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v37, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v35, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v98, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v97, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v86, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v85, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v69, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v67, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v16, 16, v15
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v20, 16, v19
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -42529,64 +42529,64 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v115.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v113.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v114.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v112.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v102.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v2.l
@@ -42594,14 +42594,14 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v97.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v26.l, 3
@@ -42621,69 +42621,69 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v24.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v86.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v82.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v114.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v80.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v68.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v112.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v84.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v50.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v83.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v102.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v69.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v99.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v96.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.h, v11.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v85.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v17.l, v15.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v17.h, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h
@@ -42693,14 +42693,14 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v11.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v12.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v12.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
@@ -55873,63 +55873,63 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -55940,15 +55940,15 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v119, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v117, v116, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v115, v114, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v113, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v103, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v102, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v99, v98, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v97, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v96, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v37, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v22, v23, 0xc0c0004
@@ -55962,50 +55962,50 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v86, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v84, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v82, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v80, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v71, v69, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v68, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v30, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v118, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v116, v115, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v114, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v112, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v103, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v100, v99, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v15, 16, v14
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v66, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v65, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v54, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v51, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v48, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v37, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v35, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v98, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v97, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v86, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v85, v82, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v80, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v69, v66, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v67, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v14, 16, v13
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v13, v16, 16, v15
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v14, v18, 16, v17
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v20, 16, v19
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -56020,64 +56020,64 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v117.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v118.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v115.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v113.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v114.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v112.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v102.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v100.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, 0x300, v2.l
@@ -56085,14 +56085,14 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.h, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v97.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v26.l, 3
@@ -56112,69 +56112,69 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v24.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v86.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v82.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v118.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v114.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v113.l
; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v80.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v68.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v112.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v103.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v7.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v87.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v97.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v87.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v84.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v84.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v12.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v50.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v83.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v82.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v102.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v69.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.h, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v99.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v65.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v10.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v96.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.h, v11.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v85.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v17.l, v15.l
; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v17.h, v15.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h
@@ -56184,14 +56184,14 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, 0x300, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v11.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v12.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v12.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v13.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v14.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v71.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v3.h
@@ -69690,91 +69690,81 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v27, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v17, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v16, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v19, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v30, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v26, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v25, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v24, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v7.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v37.h, 8, v0.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v38.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v36.h, 8, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v35.h, 8, v2.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v5.h
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB98_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -69785,274 +69775,273 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v49, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v25.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v53, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v30.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v20.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v48, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v28.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v24.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v67, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v29.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v71, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v35.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v37.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v22.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v23.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v27.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v21.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v39.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v20.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v38.h, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v37.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v19.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v36.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v16.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v35.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v69.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v31.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v33.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v34.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v25.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v29.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v24.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v18.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v28.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v22.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v23.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v87.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
@@ -71905,36 +71894,36 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
@@ -71943,56 +71932,56 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s22, s23, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s26, s27, v8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s18, s19, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s42, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s77 :: v_dual_lshlrev_b32 v15, 8, v49
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s79, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s12, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s79, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s57, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: s_or_b32 s77, s77, s78
; GFX11-TRUE16-NEXT: s_or_b32 s78, s79, s88
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v48
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v39
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s77, s78
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s78
@@ -72038,76 +72027,76 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v48
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -82479,91 +82468,81 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v27, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v17, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v16, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v19, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v30, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v26, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v25, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v24, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v7.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v37.h, 8, v0.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v38.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v36.h, 8, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v35.h, 8, v2.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v5.h
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB106_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -82574,274 +82553,273 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v49, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v25.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v53, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v30.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v20.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v48, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v28.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v24.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v67, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v29.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v71, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v35.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v37.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v22.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v23.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v27.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v21.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v39.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v20.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v38.h, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v37.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v19.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v36.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v16.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v35.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v69.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v31.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v33.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v34.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v25.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v29.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v24.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v18.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v28.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v22.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v23.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v87.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
@@ -84694,36 +84672,36 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
@@ -84732,56 +84710,56 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s22, s23, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s26, s27, v8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s18, s19, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s42, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s77 :: v_dual_lshlrev_b32 v15, 8, v49
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s79, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s12, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s79, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s57, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: s_or_b32 s77, s77, s78
; GFX11-TRUE16-NEXT: s_or_b32 s78, s79, s88
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v48
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v39
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s77, s78
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s78
@@ -84827,76 +84805,76 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v48
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -93201,91 +93179,81 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v27, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v17, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v1, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v2, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v16, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v2, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v19, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v3, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v3, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v4, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v4, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v5, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v5, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v30, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v26, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v25, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v24, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v7.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v37.h, 8, v0.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v38.h, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v36.h, 8, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v35.h, 8, v2.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v5.h
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB110_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -93296,274 +93264,273 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v49, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v25.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v53, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v30.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v20.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v48, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v28.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v24.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v67, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v29.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v71, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v35.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v33.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v37.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v22.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v23.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v70.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v27.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v21.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v39.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v20.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v38.h, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v37.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v37.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v19.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v36.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v16.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v35.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v69.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v31.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v33.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v34.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v25.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v29.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v24.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v18.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v28.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v22.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v23.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v81.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v87.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v39.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
@@ -95385,36 +95352,36 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v23
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v21
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v18
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v11
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
@@ -95423,56 +95390,56 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
-; GFX11-TRUE16-NEXT: s_and_b32 s76, s62, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s76, s56, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s77, s44, 8
; GFX11-TRUE16-NEXT: s_and_b32 s78, s43, 0xff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v5, s22, s23, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v6, s26, s27, v8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s9, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s40, 8
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s0, s1, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s18, s19, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s5, s4, v8
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v8
; GFX11-TRUE16-NEXT: s_or_b32 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s14, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s42, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s8, s6, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s11, s7, v8
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s76, s76, s77
; GFX11-TRUE16-NEXT: s_or_b32 s77, s78, s79
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s13, s5, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s14, s10, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s77 :: v_dual_lshlrev_b32 v15, 8, v49
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s13, s9, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s12, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s77, s73, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s57, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s79, s56, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s12, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s45, s41, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s77, s72, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s78, s61, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s79, s60, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s88, s57, 8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s58, s47, v8
; GFX11-TRUE16-NEXT: s_or_b32 s77, s77, s78
; GFX11-TRUE16-NEXT: s_or_b32 s78, s79, s88
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xff, v48
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 8, v39
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s77, s77, s78
-; GFX11-TRUE16-NEXT: s_and_b32 s78, s46, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s11, 8
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s62, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s79, s59, 8
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s63, v8
; GFX11-TRUE16-NEXT: s_or_b32 s78, s78, s79
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v12
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s78
@@ -95518,76 +95485,76 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s58, s58, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s61, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v48
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s63, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s46, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s40, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s58, s47, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s59, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s45, s41, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s15, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s13, s9, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s10, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s56, s44, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s5, s4, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s8, s8, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s11, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s8, s6, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 275118e33eeab..a16aae0c7a178 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2935,11 +2935,11 @@ define <8 x i8> @bitcast_i64_to_v8i8(i64 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i64_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -2948,30 +2948,30 @@ define <8 x i8> @bitcast_i64_to_v8i8(i64 %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v3, 3
+; GFX11-TRUE16-NEXT: v_add_co_u32 v8, vcc_lo, v8, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i64_to_v8i8:
@@ -3174,50 +3174,98 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_i64_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_branch .LBB25_3
-; GFX11-NEXT: .LBB25_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: .LBB25_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s8, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s8, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_u32 s0, s0, 3
-; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: .LBB25_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
-; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_i64_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB25_3
+; GFX11-TRUE16-NEXT: .LBB25_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_i64_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB25_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB25_3
+; GFX11-FAKE16-NEXT: .LBB25_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: .LBB25_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, 0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-FAKE16-NEXT: .LBB25_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3424,8 +3472,8 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -3438,12 +3486,12 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -3456,11 +3504,11 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -6360,11 +6408,11 @@ define <8 x i8> @bitcast_f64_to_v8i8(double %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f64_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -6373,28 +6421,28 @@ define <8 x i8> @bitcast_f64_to_v8i8(double %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], v[3:4], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[8:9], v[8:9], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f64_to_v8i8:
@@ -6605,54 +6653,105 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_f64_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-NEXT: s_lshr_b32 s8, s0, 16
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_branch .LBB49_3
-; GFX11-NEXT: .LBB49_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: .LBB49_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f64 v[8:9], s[0:1], 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB49_6
-; GFX11-NEXT: .LBB49_5:
-; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_mov_b32_e32 v7, s6
-; GFX11-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-NEXT: .LBB49_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_f64_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s2 :: v_dual_mov_b32 v6, s5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s3
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_f64_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f64 v[8:9], s[0:1], 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v6, s5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6859,8 +6958,8 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -6873,12 +6972,12 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -6891,11 +6990,11 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -9445,11 +9544,11 @@ define <8 x i8> @bitcast_v2i32_to_v8i8(<2 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2i32_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -9458,29 +9557,29 @@ define <8 x i8> @bitcast_v2i32_to_v8i8(<2 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 3, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i32_to_v8i8:
@@ -9682,50 +9781,98 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2i32_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_branch .LBB69_3
-; GFX11-NEXT: .LBB69_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: .LBB69_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s8, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s8, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: .LBB69_5: ; %end
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
-; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v2i32_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB69_3
+; GFX11-TRUE16-NEXT: .LBB69_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2i32_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB69_3
+; GFX11-FAKE16-NEXT: .LBB69_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-FAKE16-NEXT: .LBB69_5: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9932,8 +10079,8 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -9946,12 +10093,12 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -9964,11 +10111,11 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -12169,11 +12316,11 @@ define <8 x i8> @bitcast_v2f32_to_v8i8(<2 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2f32_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -12182,28 +12329,28 @@ define <8 x i8> @bitcast_v2f32_to_v8i8(<2 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f32_to_v8i8:
@@ -12417,55 +12564,107 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2f32_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_branch .LBB85_3
-; GFX11-NEXT: .LBB85_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: .LBB85_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v9, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v8, s0, 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB85_6
-; GFX11-NEXT: .LBB85_5:
-; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_mov_b32_e32 v7, s3
-; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB85_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v2f32_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB85_3
+; GFX11-TRUE16-NEXT: .LBB85_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB85_6
+; GFX11-TRUE16-NEXT: .LBB85_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2f32_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB85_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB85_3
+; GFX11-FAKE16-NEXT: .LBB85_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v9, s1, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v8, s0, 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-FAKE16-NEXT: s_branch .LBB85_6
+; GFX11-FAKE16-NEXT: .LBB85_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s2
+; GFX11-FAKE16-NEXT: .LBB85_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -12672,8 +12871,8 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -12686,12 +12885,12 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -12704,11 +12903,11 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -14548,11 +14747,11 @@ define <8 x i8> @bitcast_v4i16_to_v8i8(<4 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4i16_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -14561,29 +14760,29 @@ define <8 x i8> @bitcast_v4i16_to_v8i8(<4 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i16_to_v8i8:
@@ -14814,55 +15013,107 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i16_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_branch .LBB97_3
-; GFX11-NEXT: .LBB97_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: .LBB97_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v9, s1, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v8, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB97_6
-; GFX11-NEXT: .LBB97_5:
-; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_mov_b32_e32 v7, s3
-; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB97_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i16_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB97_3
+; GFX11-TRUE16-NEXT: .LBB97_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB97_6
+; GFX11-TRUE16-NEXT: .LBB97_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i16_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB97_3
+; GFX11-FAKE16-NEXT: .LBB97_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v9, s1, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v8, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-FAKE16-NEXT: s_branch .LBB97_6
+; GFX11-FAKE16-NEXT: .LBB97_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s2
+; GFX11-FAKE16-NEXT: .LBB97_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -15077,8 +15328,8 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -15091,12 +15342,12 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -15109,11 +15360,11 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -16494,11 +16745,11 @@ define <8 x i8> @bitcast_v4f16_to_v8i8(<4 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4f16_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -16507,29 +16758,29 @@ define <8 x i8> @bitcast_v4f16_to_v8i8(<4 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f16_to_v8i8:
@@ -16770,55 +17021,107 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4f16_to_v8i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_branch .LBB105_3
-; GFX11-NEXT: .LBB105_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: .LBB105_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB105_6
-; GFX11-NEXT: .LBB105_5:
-; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_mov_b32_e32 v7, s3
-; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB105_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4f16_to_v8i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_branch .LBB105_3
+; GFX11-TRUE16-NEXT: .LBB105_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB105_6
+; GFX11-TRUE16-NEXT: .LBB105_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4f16_to_v8i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB105_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB105_3
+; GFX11-FAKE16-NEXT: .LBB105_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: .LBB105_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v9, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v8, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-FAKE16-NEXT: s_branch .LBB105_6
+; GFX11-FAKE16-NEXT: .LBB105_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s2
+; GFX11-FAKE16-NEXT: .LBB105_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -17033,8 +17336,8 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -17047,12 +17350,12 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -17065,11 +17368,11 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -17649,11 +17952,11 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
@@ -17662,72 +17965,67 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8:
@@ -18087,59 +18385,64 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v7 :: v_dual_add_nc_u32 v6, v6, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v6, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v0, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v8
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
; GFX11-TRUE16-NEXT: .LBB109_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v6, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s5
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8_scalar:
@@ -18443,8 +18746,8 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -18457,12 +18760,12 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -18475,11 +18778,11 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 374626a2578fb..c421dcf7835b5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -554,8 +554,8 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v3i32_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -565,7 +565,7 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -573,35 +573,34 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB4_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 3, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB4_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3i32_to_v12i8:
@@ -856,61 +855,123 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v3i32_to_v12i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-NEXT: s_lshr_b32 s9, s1, 24
-; GFX11-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-NEXT: s_lshr_b32 s11, s1, 8
-; GFX11-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-NEXT: s_lshr_b32 s13, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB5_3
-; GFX11-NEXT: .LBB5_2:
-; GFX11-NEXT: s_mov_b32 s14, -1
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: .LBB5_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB5_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-NEXT: s_lshr_b32 s9, s1, 24
-; GFX11-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-NEXT: s_lshr_b32 s11, s1, 8
-; GFX11-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-NEXT: s_lshr_b32 s13, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: .LBB5_5: ; %end
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13
-; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v3, s4
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s11
-; GFX11-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v7, s9
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s8
-; GFX11-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v11, s6
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v3i32_to_v12i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB5_3
+; GFX11-TRUE16-NEXT: .LBB5_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB5_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: .LBB5_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s6
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v3i32_to_v12i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB5_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB5_3
+; GFX11-FAKE16-NEXT: .LBB5_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB5_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: .LBB5_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v3, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s11
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v7, s9
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v11, s6
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1179,9 +1240,9 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -1194,16 +1255,16 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB6_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v15, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -1219,9 +1280,9 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_2
; GFX11-TRUE16-NEXT: .LBB6_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -3873,8 +3934,8 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v3f32_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -3884,7 +3945,7 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -3892,34 +3953,33 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB20_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 1.0, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 1.0, v12 :: v_dual_add_f32 v11, 1.0, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 1.0, v14 :: v_dual_add_f32 v13, 1.0, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB20_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3f32_to_v12i8:
@@ -4187,67 +4247,131 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, v14
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v3f32_to_v12i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB21_3
-; GFX11-NEXT: .LBB21_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: .LBB21_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB21_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v8, s2, 1.0
-; GFX11-NEXT: v_add_f32_e64 v14, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v13, s0, 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX11-NEXT: s_branch .LBB21_6
-; GFX11-NEXT: .LBB21_5:
-; GFX11-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
-; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB21_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v13
-; GFX11-NEXT: v_mov_b32_e32 v4, v14
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v3f32_to_v12i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB21_3
+; GFX11-TRUE16-NEXT: .LBB21_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, s2, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB21_6
+; GFX11-TRUE16-NEXT: .LBB21_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB21_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v3f32_to_v12i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB21_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB21_3
+; GFX11-FAKE16-NEXT: .LBB21_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB21_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v8, s2, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v14, s1, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v13, s0, 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-FAKE16-NEXT: s_branch .LBB21_6
+; GFX11-FAKE16-NEXT: .LBB21_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB21_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v14
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4516,9 +4640,9 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -4531,16 +4655,16 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB22_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v15, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -4556,9 +4680,9 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
; GFX11-TRUE16-NEXT: .LBB22_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -7348,10 +7472,10 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -7364,17 +7488,17 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB36_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -7389,14 +7513,14 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB36_2
; GFX11-TRUE16-NEXT: .LBB36_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -8143,7 +8267,7 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v2 :: v_dual_mov_b32 v14, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v2 :: v_dual_mov_b32 v14, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
@@ -8154,19 +8278,19 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v15
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[15:16]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB38_2: ; %Flow
@@ -8176,12 +8300,12 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v14.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, 0x7fc07fc0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -8207,7 +8331,7 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v4 :: v_dual_add_f32 v4, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
@@ -8233,20 +8357,19 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v15
; GFX11-TRUE16-NEXT: .LBB38_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v15.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8:
@@ -8731,86 +8854,86 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v8, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: s_branch .LBB39_6
; GFX11-TRUE16-NEXT: .LBB39_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v1, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s13 :: v_dual_mov_b32 v5, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s12 :: v_dual_mov_b32 v11, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v9, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s13 :: v_dual_mov_b32 v7, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s12 :: v_dual_mov_b32 v13, s9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: .LBB39_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8_scalar:
@@ -9217,10 +9340,10 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -9233,17 +9356,17 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB40_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -9258,14 +9381,14 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2
; GFX11-TRUE16-NEXT: .LBB40_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -9919,8 +10042,8 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6f16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v2 :: v_dual_mov_b32 v14, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v16, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -9930,46 +10053,45 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX11-TRUE16-NEXT: .LBB42_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7e007e00
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, 0x7e007e00
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX11-TRUE16-NEXT: .LBB42_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6f16_to_v12i8:
@@ -10281,70 +10403,136 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX9-NEXT: v_mov_b32_e32 v9, v13
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v6f16_to_v12i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB43_3
-; GFX11-NEXT: .LBB43_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: .LBB43_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB43_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_mov_b32_e32 v9, 0x7e007e00
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v14
-; GFX11-NEXT: s_branch .LBB43_6
-; GFX11-NEXT: .LBB43_5:
-; GFX11-NEXT: v_dual_mov_b32 v14, s0 :: v_dual_mov_b32 v15, s1
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
-; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v13, s9 :: v_dual_mov_b32 v10, s3
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB43_6: ; %end
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v0, v14 :: v_dual_mov_b32 v9, v13
-; GFX11-NEXT: v_mov_b32_e32 v4, v15
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v6f16_to_v12i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB43_3
+; GFX11-TRUE16-NEXT: .LBB43_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB43_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, 0x7e007e00
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB43_6
+; GFX11-TRUE16-NEXT: .LBB43_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s13 :: v_dual_mov_b32 v13, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB43_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v6f16_to_v12i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB43_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB43_3
+; GFX11-FAKE16-NEXT: .LBB43_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB43_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v15, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v14, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0x7e007e00
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
+; GFX11-FAKE16-NEXT: s_branch .LBB43_6
+; GFX11-FAKE16-NEXT: .LBB43_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v14, s0 :: v_dual_mov_b32 v15, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s9 :: v_dual_mov_b32 v10, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB43_6: ; %end
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v14 :: v_dual_mov_b32 v9, v13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v15
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10629,10 +10817,10 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -10645,17 +10833,17 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB44_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -10670,14 +10858,14 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB44_2
; GFX11-TRUE16-NEXT: .LBB44_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -11329,8 +11517,8 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6i16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
@@ -11340,7 +11528,7 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -11348,37 +11536,36 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB46_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, v0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8:
@@ -11674,69 +11861,134 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v4, v14
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v6i16_to_v12i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_branch .LBB47_3
-; GFX11-NEXT: .LBB47_2:
-; GFX11-NEXT: s_mov_b32 s8, -1
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: .LBB47_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB47_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v14, s1, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v9, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v13, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX11-NEXT: s_branch .LBB47_6
-; GFX11-NEXT: .LBB47_5:
-; GFX11-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
-; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
-; GFX11-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB47_6: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, v13
-; GFX11-NEXT: v_mov_b32_e32 v4, v14
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v6i16_to_v12i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_branch .LBB47_3
+; GFX11-TRUE16-NEXT: .LBB47_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB47_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB47_6
+; GFX11-TRUE16-NEXT: .LBB47_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s13 :: v_dual_mov_b32 v9, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB47_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB47_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-FAKE16-NEXT: s_branch .LBB47_3
+; GFX11-FAKE16-NEXT: .LBB47_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB47_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v14, s1, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v9, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v13, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-FAKE16-NEXT: s_branch .LBB47_6
+; GFX11-FAKE16-NEXT: .LBB47_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: .LBB47_6: ; %end
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v13
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v14
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index f03c9195edbfa..3921b453cf8e1 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -49,9 +49,9 @@ define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -74,9 +74,9 @@ define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index e289b558aecdf..fc93097ac50db 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -600,101 +600,101 @@ define <4 x float> @v4_from_e5m3fnu(<4 x i8> %x) {
; GFX1170-NEXT: v_or_b32_e32 v10, v0, v4
; GFX1170-NEXT: v_cmp_eq_u32_e64 s1, 7, v4
; GFX1170-NEXT: v_sub_nc_u32_e32 v11, 31, v8
-; GFX1170-NEXT: v_add_nc_u32_e32 v17, -8, v8
+; GFX1170-NEXT: v_add_nc_u32_e32 v16, -8, v8
; GFX1170-NEXT: v_cmp_eq_u32_e64 s0, 0, v0
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 31, v0
; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 0x8d, v8
; GFX1170-NEXT: v_lshlrev_b32_e64 v11, v11, 1
; GFX1170-NEXT: v_lshl_or_b32 v0, v0, 23, v9
; GFX1170-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
-; GFX1170-NEXT: v_clz_i32_u32_e32 v15, v6
; GFX1170-NEXT: s_and_b32 s0, s2, s1
+; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v6
; GFX1170-NEXT: v_xor_b32_e32 v4, v4, v11
; GFX1170-NEXT: v_add_nc_u32_e32 v0, 0x38000000, v0
-; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v6
+; GFX1170-NEXT: v_clz_i32_u32_e32 v11, v6
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 0, v2
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v17, v4
+; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v16, v4
; GFX1170-NEXT: s_and_b32 s1, s2, s1
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 31, v2
; GFX1170-NEXT: v_lshl_or_b32 v4, v8, 23, v4
; GFX1170-NEXT: v_and_b32_e32 v5, 7, v1
; GFX1170-NEXT: v_bfe_u32 v1, v1, 3, 5
-; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 31, v15
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
; GFX1170-NEXT: v_clz_i32_u32_e32 v12, v5
-; GFX1170-NEXT: v_lshlrev_b32_e32 v13, 20, v5
-; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v10
-; GFX1170-NEXT: v_and_b32_e32 v7, 7, v3
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v5
-; GFX1170-NEXT: v_sub_nc_u32_e32 v16, 31, v12
-; GFX1170-NEXT: v_add_nc_u32_e32 v9, -8, v12
-; GFX1170-NEXT: v_sub_nc_u32_e32 v12, 0x8d, v12
-; GFX1170-NEXT: v_lshl_or_b32 v13, v1, 23, v13
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v1
-; GFX1170-NEXT: v_lshlrev_b32_e64 v16, v16, 1
-; GFX1170-NEXT: v_or_b32_e32 v14, v1, v5
-; GFX1170-NEXT: v_lshlrev_b32_e64 v4, v8, 1
-; GFX1170-NEXT: v_add_nc_u32_e32 v8, 0x38000000, v13
+; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v10
+; GFX1170-NEXT: v_and_b32_e32 v7, 7, v3
+; GFX1170-NEXT: v_sub_nc_u32_e32 v15, 31, v12
+; GFX1170-NEXT: v_add_nc_u32_e32 v16, -8, v12
+; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 0x8d, v12
; GFX1170-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX1170-NEXT: v_xor_b32_e32 v16, v5, v16
-; GFX1170-NEXT: v_clz_i32_u32_e32 v11, v7
; GFX1170-NEXT: s_and_b32 vcc_lo, s4, s3
-; GFX1170-NEXT: v_xor_b32_e32 v4, v6, v4
+; GFX1170-NEXT: v_lshlrev_b32_e64 v15, v15, 1
+; GFX1170-NEXT: v_lshlrev_b32_e32 v13, 20, v5
+; GFX1170-NEXT: v_sub_nc_u32_e32 v4, 31, v11
+; GFX1170-NEXT: v_or_b32_e32 v14, v1, v5
+; GFX1170-NEXT: v_clz_i32_u32_e32 v9, v7
+; GFX1170-NEXT: v_xor_b32_e32 v15, v5, v15
+; GFX1170-NEXT: v_lshl_or_b32 v12, v1, 23, v13
+; GFX1170-NEXT: v_lshlrev_b32_e64 v4, v4, 1
+; GFX1170-NEXT: v_add_nc_u32_e32 v10, -8, v11
; GFX1170-NEXT: v_cndmask_b32_e64 v0, v0, 0x7fc00000, s0
-; GFX1170-NEXT: v_lshlrev_b32_e32 v9, v9, v16
-; GFX1170-NEXT: v_sub_nc_u32_e32 v10, 31, v11
+; GFX1170-NEXT: v_lshlrev_b32_e32 v13, v16, v15
+; GFX1170-NEXT: v_add_nc_u32_e32 v12, 0x38000000, v12
+; GFX1170-NEXT: v_xor_b32_e32 v4, v6, v4
; GFX1170-NEXT: v_cmp_eq_u32_e64 s0, 31, v1
; GFX1170-NEXT: v_bfe_u32 v3, v3, 3, 5
+; GFX1170-NEXT: v_lshl_or_b32 v8, v8, 23, v13
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v7
-; GFX1170-NEXT: v_lshl_or_b32 v9, v12, 23, v9
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
-; GFX1170-NEXT: v_dual_cndmask_b32 v8, v8, v9 :: v_dual_add_nc_u32 v9, -8, v15
+; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v10, v4
+; GFX1170-NEXT: v_sub_nc_u32_e32 v10, 0x8d, v11
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: v_dual_cndmask_b32 v8, v12, v8 :: v_dual_lshlrev_b32 v11, 20, v6
+; GFX1170-NEXT: v_sub_nc_u32_e32 v12, 31, v9
; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v9, v4
+; GFX1170-NEXT: v_lshl_or_b32 v4, v10, 23, v4
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1170-NEXT: v_lshl_or_b32 v10, v2, 23, v11
+; GFX1170-NEXT: v_lshlrev_b32_e32 v11, 20, v7
+; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
; GFX1170-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
; GFX1170-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v5
-; GFX1170-NEXT: v_lshlrev_b32_e64 v5, v10, 1
-; GFX1170-NEXT: v_sub_nc_u32_e32 v9, 0x8d, v15
-; GFX1170-NEXT: v_lshlrev_b32_e32 v10, 20, v6
+; GFX1170-NEXT: v_lshlrev_b32_e64 v5, v12, 1
+; GFX1170-NEXT: v_add_nc_u32_e32 v10, 0x38000000, v10
; GFX1170-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_xor_b32_e32 v1, v7, v5
-; GFX1170-NEXT: v_add_nc_u32_e32 v5, -8, v11
-; GFX1170-NEXT: v_lshl_or_b32 v4, v9, 23, v4
-; GFX1170-NEXT: v_lshl_or_b32 v9, v2, 23, v10
-; GFX1170-NEXT: v_lshlrev_b32_e32 v10, 20, v7
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, v5, v1
-; GFX1170-NEXT: v_sub_nc_u32_e32 v5, 0x8d, v11
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1170-NEXT: v_add_nc_u32_e32 v9, 0x38000000, v9
-; GFX1170-NEXT: v_lshl_or_b32 v10, v3, 23, v10
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_lshl_or_b32 v1, v5, 23, v1
-; GFX1170-NEXT: v_cndmask_b32_e64 v4, v9, v4, s1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_add_nc_u32_e32 v5, 0x38000000, v10
-; GFX1170-NEXT: v_or_b32_e32 v10, v2, v6
+; GFX1170-NEXT: v_add_nc_u32_e32 v5, -8, v9
+; GFX1170-NEXT: v_cndmask_b32_e64 v4, v10, v4, s1
; GFX1170-NEXT: s_and_b32 s1, s4, s3
; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 31, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, v5, v1
+; GFX1170-NEXT: v_sub_nc_u32_e32 v5, 0x8d, v9
+; GFX1170-NEXT: v_lshl_or_b32 v9, v3, 23, v11
+; GFX1170-NEXT: v_lshl_or_b32 v1, v5, 23, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_add_nc_u32_e32 v5, 0x38000000, v9
+; GFX1170-NEXT: v_or_b32_e32 v9, v2, v6
; GFX1170-NEXT: v_cndmask_b32_e64 v1, v5, v1, s1
; GFX1170-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v10
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v9
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v5
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_cndmask_b32_e64 v4, 0, v4, s1
; GFX1170-NEXT: v_cmp_eq_u32_e64 s1, 7, v6
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v5, 0, v1, s3
; GFX1170-NEXT: v_cmp_eq_u32_e64 s3, 7, v7
; GFX1170-NEXT: v_cndmask_b32_e64 v1, v8, 0x7fc00000, s0
; GFX1170-NEXT: s_and_b32 s0, s2, s1
+; GFX1170-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v2, v4, 0x7fc00000, s0
; GFX1170-NEXT: s_and_b32 s0, s4, s3
-; GFX1170-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v3, v5, 0x7fc00000, s0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index d3e89cd1e469b..683207fa1d273 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -2796,33 +2796,18 @@ define bfloat @test_byval(ptr addrspace(5) byval(bfloat) %bv, bfloat %val) #0 {
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], s32
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: test_byval:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11TRUE16-NEXT: scratch_store_b16 off, v1, s32
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: test_byval:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: scratch_store_b16 off, v0, s32
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: test_byval:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX1250TRUE16-NEXT: scratch_store_b16 off, v1, s32
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: test_byval:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: scratch_store_b16 off, v0, s32
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: test_byval:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: scratch_store_b16 off, v0, s32
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: test_byval:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: scratch_store_b16 off, v0, s32
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
store bfloat %val, ptr addrspace(5) %bv
%retval = load bfloat, ptr addrspace(5) %bv
ret bfloat %retval
@@ -5384,93 +5369,48 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) #0
; GFX10-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen offset:128
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: test_overflow_stack:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: s_clause 0x2
-; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11TRUE16-NEXT: s_clause 0x3
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX11TRUE16-NEXT: s_clause 0x1
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: s_clause 0x2
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX11TRUE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: test_overflow_stack:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: s_clause 0x2
-; GFX11FAKE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11FAKE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11FAKE16-NEXT: s_clause 0x5
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11FAKE16-NEXT: s_clause 0x2
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX11FAKE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: test_overflow_stack:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: s_clause 0x2
-; GFX1250TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX1250TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX1250TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX1250TRUE16-NEXT: s_clause 0x3
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX1250TRUE16-NEXT: s_clause 0x1
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250TRUE16-NEXT: s_clause 0x2
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX1250TRUE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: test_overflow_stack:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-NEXT: s_clause 0x5
+; GFX11-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
+; GFX11-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
+; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
+; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
+; GFX11-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
+; GFX11-NEXT: scratch_store_b16 v0, v1, off offset:128
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: test_overflow_stack:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: s_clause 0x2
-; GFX1250FAKE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX1250FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX1250FAKE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX1250FAKE16-NEXT: s_clause 0x5
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX1250FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250FAKE16-NEXT: s_clause 0x2
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX1250FAKE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: test_overflow_stack:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX1250-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX1250-NEXT: scratch_load_b32 v31, off, s32
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
+; GFX1250-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
+; GFX1250-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
+; GFX1250-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
+; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
+; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
+; GFX1250-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
+; GFX1250-NEXT: scratch_store_b16 v0, v1, off offset:128
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ins.0 = insertvalue { <32 x i32>, bfloat } poison, <32 x i32> %b, 0
%ins.1 = insertvalue { <32 x i32>, bfloat } %ins.0 ,bfloat %a, 1
ret { <32 x i32>, bfloat } %ins.1
@@ -29207,16 +29147,29 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) #0 {
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_frexp_bf16_i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX1250-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_frexp_bf16_i16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v0
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v1, s0
+; GFX1250TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_frexp_bf16_i16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250FAKE16-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX1250FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = call { bfloat, i16 } @llvm.frexp.bf16.i16(bfloat %a)
ret { bfloat, i16 } %op
}
@@ -30022,31 +29975,28 @@ define bfloat @v_exp_bf16(bfloat %a) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x3fb8aa3b
-; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v0, s0, neg(0) op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, -v1 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v3, v1
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x32a5705f
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -30435,31 +30385,28 @@ define bfloat @v_exp10_bf16(bfloat %a) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x40549a78
-; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v0, s0, neg(0) op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, -v1 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v3, v1
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x33979a37
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -31011,23 +30958,42 @@ define bfloat @v_round_bf16(bfloat %a) #0 {
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_round_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_round_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_round_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250FAKE16-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
}
@@ -35940,27 +35906,49 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) #0 {
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_sitofp_i64_to_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cls_i32_e32 v3, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14
-; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 31, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u32_e32 v2, 32, v2
-; GFX1250-NEXT: v_min_u32_e32 v2, v3, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
-; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_sitofp_i64_to_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_cls_i32_e32 v3, v1
+; GFX1250TRUE16-NEXT: v_xor_b32_e32 v2, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_ashrrev_i32 v2, 31, v2
+; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_sitofp_i64_to_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_cls_i32_e32 v3, v1
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14
+; GFX1250FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v2
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
+; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = sitofp i64 %x to bfloat
ret bfloat %op
}
@@ -42859,28 +42847,28 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v5.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v7.l
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v9
+; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v4.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v14
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v4.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v1.h
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v1.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
-; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.h
+; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v7.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v6.l
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v11
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v0.l
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v10
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.l
-; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v6.l
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v9
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v13
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v8
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v12
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.h
; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, v1.l, s5
; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v4.l, v3.l, s4
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v5.l, s2
@@ -42939,28 +42927,28 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v5.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v7.l
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v5, 16, v9 :: v_dual_lshrrev_b32 v7, 16, v8
+; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v4.l
+; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v5.l
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v9
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v4.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v1.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v1.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
-; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v7.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v6.l
; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v1, 16, v11 :: v_dual_lshrrev_b32 v3, 16, v10
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v0.l
; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v2, 16, v15 :: v_dual_lshrrev_b32 v4, 16, v14
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.l
-; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v6.l
; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v6, 16, v13 :: v_dual_lshrrev_b32 v16, 16, v12
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.h
; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, v1.l, s5
; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v4.l, v3.l, s4
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.l
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v5.l, s2
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v16.l, v7.l, s0
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v12.l, v8.l, vcc_lo
@@ -43477,6 +43465,7 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
@@ -43490,24 +43479,24 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v23
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v22
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v30
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v21
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v29
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v20
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v28
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v19
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v27
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v18
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v26
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v17
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v25
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v16
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v24
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v22
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v30
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v23
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v21
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v20
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v26
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v16
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v24
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
@@ -43519,30 +43508,29 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v5.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v34.l, v33.l, s10
+; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v10.l, v9.l, s10
; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v30.l, v22.l, s11
-; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v36.l, v35.l, s12
+; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v12.l, v11.l, s12
; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v29.l, v21.l, s9
-; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v38.l, v37.l, s8
+; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v13.l, s8
; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v28.l, v20.l, s7
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v48.l, v39.l, s6
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v32.l, v15.l, s6
; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v27.l, v19.l, s5
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v50.l, v49.l, s4
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v52.l, v51.l, s2
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v54.l, v53.l, s0
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v34.l, v33.l, s4
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v36.l, v35.l, s2
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v38.l, v37.l, s0
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v24.l, v16.l, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v25.l, v17.l, s1
; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v26.l, v18.l, s3
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v31
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v31
; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v31.l, v23.l, s13
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v32.l, s14
+; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v9.l, v8.l, s14
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_vselect_v16bf16:
@@ -43630,6 +43618,7 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX1250TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
@@ -43643,17 +43632,17 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
; GFX1250TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
; GFX1250TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX1250TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX1250TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
; GFX1250TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v32, 16, v23 :: v_dual_lshrrev_b32 v33, 16, v22
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v34, 16, v30 :: v_dual_lshrrev_b32 v35, 16, v21
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v36, 16, v29 :: v_dual_lshrrev_b32 v37, 16, v20
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v38, 16, v28 :: v_dual_lshrrev_b32 v39, 16, v19
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v48, 16, v27 :: v_dual_lshrrev_b32 v49, 16, v18
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v50, 16, v26 :: v_dual_lshrrev_b32 v51, 16, v17
-; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v52, 16, v25 :: v_dual_lshrrev_b32 v53, 16, v16
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v24
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v8, 16, v23 :: v_dual_lshrrev_b32 v9, 16, v22
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v10, 16, v30 :: v_dual_lshrrev_b32 v11, 16, v21
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v12, 16, v29 :: v_dual_lshrrev_b32 v13, 16, v20
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v14, 16, v28 :: v_dual_lshrrev_b32 v15, 16, v19
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v32, 16, v27 :: v_dual_lshrrev_b32 v33, 16, v18
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v34, 16, v26 :: v_dual_lshrrev_b32 v35, 16, v17
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v36, 16, v25 :: v_dual_lshrrev_b32 v37, 16, v16
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v24
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
@@ -43665,30 +43654,29 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v5.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v34.l, v33.l, s10
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v10.l, v9.l, s10
; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.l, v30.l, v22.l, s11
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v36.l, v35.l, s12
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v12.l, v11.l, s12
; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.l, v29.l, v21.l, s9
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v38.l, v37.l, s8
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v13.l, s8
; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.l, v28.l, v20.l, s7
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v48.l, v39.l, s6
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v32.l, v15.l, s6
; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.l, v27.l, v19.l, s5
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v50.l, v49.l, s4
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v52.l, v51.l, s2
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v34.l, v33.l, s4
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v36.l, v35.l, s2
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v24.l, v16.l, vcc_lo
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v25.l, v17.l, s1
; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v26.l, v18.l, s3
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v31
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v54.l, v53.l, s0
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v31
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v38.l, v37.l, s0
; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.l, v31.l, v23.l, s13
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v32.l, s14
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v9.l, v8.l, s14
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250FAKE16-LABEL: v_vselect_v16bf16:
@@ -45028,184 +45016,184 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: s_clause 0x1f
-; GFX11TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:64
-; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:128
-; GFX11TRUE16-NEXT: scratch_load_b32 v34, off, s32 offset:60
-; GFX11TRUE16-NEXT: scratch_load_b32 v35, off, s32 offset:124
-; GFX11TRUE16-NEXT: scratch_load_b32 v36, off, s32 offset:56
-; GFX11TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:120
-; GFX11TRUE16-NEXT: scratch_load_b32 v38, off, s32 offset:52
-; GFX11TRUE16-NEXT: scratch_load_b32 v39, off, s32 offset:116
-; GFX11TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:48
-; GFX11TRUE16-NEXT: scratch_load_b32 v49, off, s32 offset:112
-; GFX11TRUE16-NEXT: scratch_load_b32 v50, off, s32 offset:44
-; GFX11TRUE16-NEXT: scratch_load_b32 v51, off, s32 offset:108
-; GFX11TRUE16-NEXT: scratch_load_b32 v52, off, s32 offset:40
-; GFX11TRUE16-NEXT: scratch_load_b32 v53, off, s32 offset:104
-; GFX11TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:36
-; GFX11TRUE16-NEXT: scratch_load_b32 v55, off, s32 offset:100
-; GFX11TRUE16-NEXT: scratch_load_b32 v64, off, s32 offset:32
-; GFX11TRUE16-NEXT: scratch_load_b32 v65, off, s32 offset:96
-; GFX11TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:28
-; GFX11TRUE16-NEXT: scratch_load_b32 v67, off, s32 offset:92
-; GFX11TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:24
-; GFX11TRUE16-NEXT: scratch_load_b32 v69, off, s32 offset:88
-; GFX11TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:20
-; GFX11TRUE16-NEXT: scratch_load_b32 v71, off, s32 offset:84
-; GFX11TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:16
-; GFX11TRUE16-NEXT: scratch_load_b32 v81, off, s32 offset:80
-; GFX11TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:12
-; GFX11TRUE16-NEXT: scratch_load_b32 v83, off, s32 offset:76
-; GFX11TRUE16-NEXT: scratch_load_b32 v84, off, s32 offset:8
-; GFX11TRUE16-NEXT: scratch_load_b32 v85, off, s32 offset:72
-; GFX11TRUE16-NEXT: scratch_load_b32 v86, off, s32 offset:4
-; GFX11TRUE16-NEXT: scratch_load_b32 v87, off, s32 offset:68
+; GFX11TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32
+; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:64
+; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:128
+; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:60
+; GFX11TRUE16-NEXT: scratch_load_b32 v34, off, s32 offset:124
+; GFX11TRUE16-NEXT: scratch_load_b32 v35, off, s32 offset:56
+; GFX11TRUE16-NEXT: scratch_load_b32 v36, off, s32 offset:120
+; GFX11TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:52
+; GFX11TRUE16-NEXT: scratch_load_b32 v38, off, s32 offset:116
+; GFX11TRUE16-NEXT: scratch_load_b32 v39, off, s32 offset:48
+; GFX11TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:112
+; GFX11TRUE16-NEXT: scratch_load_b32 v49, off, s32 offset:44
+; GFX11TRUE16-NEXT: scratch_load_b32 v50, off, s32 offset:108
+; GFX11TRUE16-NEXT: scratch_load_b32 v51, off, s32 offset:40
+; GFX11TRUE16-NEXT: scratch_load_b32 v52, off, s32 offset:104
+; GFX11TRUE16-NEXT: scratch_load_b32 v53, off, s32 offset:36
+; GFX11TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:100
+; GFX11TRUE16-NEXT: scratch_load_b32 v55, off, s32 offset:32
+; GFX11TRUE16-NEXT: scratch_load_b32 v64, off, s32 offset:96
+; GFX11TRUE16-NEXT: scratch_load_b32 v65, off, s32 offset:28
+; GFX11TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:92
+; GFX11TRUE16-NEXT: scratch_load_b32 v67, off, s32 offset:24
+; GFX11TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:88
+; GFX11TRUE16-NEXT: scratch_load_b32 v69, off, s32 offset:20
+; GFX11TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:84
+; GFX11TRUE16-NEXT: scratch_load_b32 v71, off, s32 offset:16
+; GFX11TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:80
+; GFX11TRUE16-NEXT: scratch_load_b32 v81, off, s32 offset:12
+; GFX11TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:76
+; GFX11TRUE16-NEXT: scratch_load_b32 v83, off, s32 offset:8
+; GFX11TRUE16-NEXT: scratch_load_b32 v84, off, s32 offset:72
+; GFX11TRUE16-NEXT: scratch_load_b32 v85, off, s32 offset:4
+; GFX11TRUE16-NEXT: scratch_load_b32 v86, off, s32 offset:68
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(32)
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v3.l
-; GFX11TRUE16-NEXT: v_and_b16 v2.l, 1, v4.l
-; GFX11TRUE16-NEXT: v_and_b16 v2.h, 1, v5.l
-; GFX11TRUE16-NEXT: v_and_b16 v3.l, 1, v6.l
-; GFX11TRUE16-NEXT: v_and_b16 v3.h, 1, v7.l
-; GFX11TRUE16-NEXT: v_and_b16 v4.l, 1, v8.l
-; GFX11TRUE16-NEXT: v_and_b16 v4.h, 1, v9.l
-; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
-; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
-; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
-; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
-; GFX11TRUE16-NEXT: v_and_b16 v8.l, 1, v16.l
-; GFX11TRUE16-NEXT: v_and_b16 v9.l, 1, v18.l
-; GFX11TRUE16-NEXT: v_and_b16 v10.l, 1, v20.l
-; GFX11TRUE16-NEXT: v_and_b16 v11.l, 1, v22.l
-; GFX11TRUE16-NEXT: v_and_b16 v12.l, 1, v24.l
-; GFX11TRUE16-NEXT: v_and_b16 v13.l, 1, v26.l
-; GFX11TRUE16-NEXT: v_and_b16 v14.l, 1, v28.l
-; GFX11TRUE16-NEXT: v_and_b16 v15.l, 1, v30.l
-; GFX11TRUE16-NEXT: v_and_b16 v8.h, 1, v17.l
-; GFX11TRUE16-NEXT: v_and_b16 v9.h, 1, v19.l
-; GFX11TRUE16-NEXT: v_and_b16 v10.h, 1, v21.l
-; GFX11TRUE16-NEXT: v_and_b16 v11.h, 1, v23.l
-; GFX11TRUE16-NEXT: v_and_b16 v12.h, 1, v25.l
-; GFX11TRUE16-NEXT: v_and_b16 v13.h, 1, v27.l
-; GFX11TRUE16-NEXT: v_and_b16 v14.h, 1, v29.l
+; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v14.l
+; GFX11TRUE16-NEXT: v_and_b16 v8.h, 1, v16.l
+; GFX11TRUE16-NEXT: v_and_b16 v9.h, 1, v18.l
+; GFX11TRUE16-NEXT: v_and_b16 v10.h, 1, v20.l
+; GFX11TRUE16-NEXT: v_and_b16 v11.h, 1, v22.l
+; GFX11TRUE16-NEXT: v_and_b16 v12.h, 1, v24.l
+; GFX11TRUE16-NEXT: v_and_b16 v13.h, 1, v26.l
+; GFX11TRUE16-NEXT: v_and_b16 v14.h, 1, v28.l
+; GFX11TRUE16-NEXT: v_and_b16 v15.h, 1, v30.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX11TRUE16-NEXT: v_and_b16 v2.l, 1, v3.l
+; GFX11TRUE16-NEXT: v_and_b16 v2.h, 1, v4.l
+; GFX11TRUE16-NEXT: v_and_b16 v3.l, 1, v5.l
+; GFX11TRUE16-NEXT: v_and_b16 v3.h, 1, v6.l
+; GFX11TRUE16-NEXT: v_and_b16 v4.l, 1, v7.l
+; GFX11TRUE16-NEXT: v_and_b16 v4.h, 1, v8.l
+; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v9.l
+; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v10.l
+; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v11.l
+; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v12.l
+; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v13.l
+; GFX11TRUE16-NEXT: v_and_b16 v8.l, 1, v15.l
+; GFX11TRUE16-NEXT: v_and_b16 v9.l, 1, v17.l
+; GFX11TRUE16-NEXT: v_and_b16 v10.l, 1, v19.l
+; GFX11TRUE16-NEXT: v_and_b16 v11.l, 1, v21.l
+; GFX11TRUE16-NEXT: v_and_b16 v12.l, 1, v23.l
+; GFX11TRUE16-NEXT: v_and_b16 v13.l, 1, v25.l
+; GFX11TRUE16-NEXT: v_and_b16 v14.l, 1, v27.l
+; GFX11TRUE16-NEXT: v_and_b16 v15.l, 1, v29.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v8.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v9.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v10.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v11.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v12.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v13.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 1, v14.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v15.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v1.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v2.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v2.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v3.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v3.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v5.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v6.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 1, v8.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 1, v9.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 1, v10.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 1, v11.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 1, v12.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v13.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v14.h
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(32)
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v31.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v8.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v9.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v10.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v11.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v12.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v13.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 1, v14.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v15.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v1.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v2.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v2.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v3.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v3.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v4.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v5.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v7.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v8.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 1, v9.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 1, v10.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 1, v11.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 1, v12.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 1, v13.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v14.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v15.l
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v0.h
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(31)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v31
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(30)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v33
-; GFX11TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v32.l, s28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: v_cndmask_b16 v15.l, v32.l, v31.l, s28
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v33
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(28)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v35
-; GFX11TRUE16-NEXT: v_cndmask_b16 v14.l, v35.l, v34.l, s27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v34
+; GFX11TRUE16-NEXT: v_cndmask_b16 v14.l, v34.l, v33.l, s27
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v36
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v35
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(26)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v37
-; GFX11TRUE16-NEXT: v_cndmask_b16 v13.l, v37.l, v36.l, s25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v36
+; GFX11TRUE16-NEXT: v_cndmask_b16 v13.l, v36.l, v35.l, s25
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v38
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v39
-; GFX11TRUE16-NEXT: v_cndmask_b16 v12.l, v39.l, v38.l, s23
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v38
+; GFX11TRUE16-NEXT: v_cndmask_b16 v12.l, v38.l, v37.l, s23
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(23)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v48
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v39
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(22)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v49
-; GFX11TRUE16-NEXT: v_cndmask_b16 v11.l, v49.l, v48.l, s21
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v48
+; GFX11TRUE16-NEXT: v_cndmask_b16 v11.l, v48.l, v39.l, s21
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(21)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v50
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v49
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v51
-; GFX11TRUE16-NEXT: v_cndmask_b16 v10.l, v51.l, v50.l, s19
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v50
+; GFX11TRUE16-NEXT: v_cndmask_b16 v10.l, v50.l, v49.l, s19
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(19)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v52
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v51
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(18)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v53
-; GFX11TRUE16-NEXT: v_cndmask_b16 v9.l, v53.l, v52.l, s17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v52
+; GFX11TRUE16-NEXT: v_cndmask_b16 v9.l, v52.l, v51.l, s17
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(17)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v54
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v53
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(16)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v55
-; GFX11TRUE16-NEXT: v_cndmask_b16 v8.l, v55.l, v54.l, s15
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v54
+; GFX11TRUE16-NEXT: v_cndmask_b16 v8.l, v54.l, v53.l, s15
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(15)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v64
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v55
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(14)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v65
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v64
+; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v64.l, v55.l, s13
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v66
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v65
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v67
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v66
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v68
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v67
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v69
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v68
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v70
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v69
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v71
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v70
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v80
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v71
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v81
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v80
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v82
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v81
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v83
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v82
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v84
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v83
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v85
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v84
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v86
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v85
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v87
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v87.l, v86.l, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v86
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v86.l, v85.l, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v65.l, v64.l, s13
-; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v67.l, v66.l, s11
-; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v69.l, v68.l, s9
-; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v71.l, v70.l, s7
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v81.l, v80.l, s5
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v83.l, v82.l, s3
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v85.l, v84.l, s1
+; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v66.l, v65.l, s11
+; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v68.l, v67.l, s9
+; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v70.l, v69.l, s7
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v80.l, v71.l, s5
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v82.l, v81.l, s3
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v84.l, v83.l, s1
; GFX11TRUE16-NEXT: v_cndmask_b16 v14.h, v19.l, v18.l, s29
; GFX11TRUE16-NEXT: v_cndmask_b16 v13.h, v21.l, v20.l, s26
; GFX11TRUE16-NEXT: v_cndmask_b16 v12.h, v23.l, v22.l, s24
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index a5da16543daf6..2caf9c262977d 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -786,14 +786,25 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0x96
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_xor3_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_xor3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_xor3_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_xor3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_xor3_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%xor1 = xor i16 %a, %b
%xor2 = xor i16 %xor1, %c
%ret_cast = bitcast i16 %xor2 to half
@@ -829,14 +840,25 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xfe
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_or3_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_or3_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_or3_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%or1 = or i16 %a, %b
%or2 = or i16 %or1, %c
%ret_cast = bitcast i16 %or2 to half
@@ -872,14 +894,25 @@ define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xec
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_and_or_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_and_or_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_and_or_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%and1 = and i16 %a, %b
%or1 = or i16 %and1, %c
%ret_cast = bitcast i16 %or1 to half
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 4719d6edbaaa5..fe4224f5570f6 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4966,63 +4966,120 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5067,57 +5124,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -5372,62 +5480,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5471,56 +5635,106 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index cecc14be3c521..a72f68eb3973e 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4095,63 +4095,120 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4196,57 +4253,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4503,62 +4611,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4602,56 +4766,106 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 7859ac4841a10..09bbbc3d84566 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4095,63 +4095,120 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4196,57 +4253,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4503,62 +4611,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4602,56 +4766,106 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index b5e4c0d7fad3b..65218bec1b145 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -6466,37 +6466,80 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
; SDAG-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: test_call_external_void_func_v16i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_getpc_b64 s[2:3]
-; GFX11-NEXT: s_add_u32 s2, s2, external_void_func_v16i8 at rel32@lo+4
-; GFX11-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
-; GFX11-NEXT: s_mov_b32 s32, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
-; GFX11-NEXT: s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
-; GFX11-NEXT: v_mov_b32_e32 v2, v17
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v16i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: s_getpc_b64 s[2:3]
+; GFX11-TRUE16-NEXT: s_add_u32 s2, s2, external_void_func_v16i8 at rel32@lo+4
+; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, 0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-TRUE16-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[2:3]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v16i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT: s_getpc_b64 s[2:3]
+; GFX11-FAKE16-NEXT: s_add_u32 s2, s2, external_void_func_v16i8 at rel32@lo+4
+; GFX11-FAKE16-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, 0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-FAKE16-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v17
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[2:3]
+; GFX11-FAKE16-NEXT: s_endpgm
;
; HSA-LABEL: test_call_external_void_func_v16i8:
; HSA: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index ac1230e3f6cb7..b532dc47ead13 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -3156,25 +3156,25 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v8i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v6.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 3, v7.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v2.h, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
@@ -3220,23 +3220,23 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 2, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v3.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v3.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.l, v0.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
@@ -3379,53 +3379,53 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v16i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, v10.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 1, v9.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, v8.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v10.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v13.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v12.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v9.l
+; GFX11-TRUE16-NEXT: v_and_b16 v6.h, v8.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v6.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, v14.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 1, v13.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, v12.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v8.l, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v14.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 2, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v12.l, v13.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v3.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.h, v10.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v8.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v7.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 15
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 4, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -3493,40 +3493,40 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v10.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v8.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 2, v2.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v10.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v14.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v9.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v11.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 2, v3.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v8.l, 1
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v13.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v2.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v5.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v14.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v9.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 2, v5.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v1.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v3.h, 3, v2.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v3.h, v5.h, v4.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.l, v2.l, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v5.l, 3, v3.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v3.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v4.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v12.l, v6.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v5.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v2.h, 15, v3.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.h, v4.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.l, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v3.h, 15, v3.l bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.h, v4.l, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
@@ -3782,94 +3782,94 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v32i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v26.l, v26.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 1, v25.l
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, v24.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v20.h, v22.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 1, v21.l
-; GFX11-TRUE16-NEXT: v_and_b16 v20.l, v20.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v17.h, v18.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 1, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, v16.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, v10.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v30.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v29.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v28.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v24.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v26.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v21.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v27.l
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 2, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v22.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v20.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 1, v17.l
+; GFX11-TRUE16-NEXT: v_and_b16 v9.h, v16.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v6.h
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v18.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v8.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 3, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 2, v7.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v8.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v6.h, v7.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v13.l
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v12.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 1, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, v8.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v3.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v10.l, 1
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, v30.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 1, v29.l
-; GFX11-TRUE16-NEXT: v_and_b16 v28.l, v28.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 3, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 2, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v24.l, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 3, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 2, v20.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v20.l, v21.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 2, v17.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, v14.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 1, v13.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, v12.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v8.l, v9.l
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v14.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 3, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v9.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 2, v6.l
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 3, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 2, v30.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v29.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_and_b16 v21.h, v22.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v22.l, v20.h
-; GFX11-TRUE16-NEXT: v_and_b16 v16.h, v16.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v17.h
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 2, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v12.l, v13.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v8.h, v4.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v7.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v6.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v28.h, v29.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.h, v28.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v21.h, v25.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v16.h, v18.l
-; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v17.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.h, v10.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v8.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v26.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.h, v19.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 4, v16.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v15.h, 15
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v9.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 15
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 4, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v3.h, 15
+; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 4, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 12, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 12, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 12, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 12, v3.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v2.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
@@ -3983,79 +3983,79 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v19.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v19.l, v22.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v18.l, v18.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.h, 3, v23.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.h, 1, v21.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 1, v17.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 2, v19.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.l, 2, v18.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.l, v16.l, v17.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v16.h, v17.h, v19.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.h, v20.l, v19.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_and_b16 v17.l, v26.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v22.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v18.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v23.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v19.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v21.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v25.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v6.h, v24.l, 1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v20.l, v4.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v26.l, 1
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v17.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v27.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v8.h, v30.l, 1
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 2, v3.h
-; GFX1250-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v18.h, v24.l, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.h, v17.h, v16.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.h, 1, v25.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 3, v27.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 2, v17.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v19.h, v30.l, 1
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v3.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v5.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v10.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v14.l, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.h, v18.h, 3, v17.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v17.l, v19.l, v17.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v31.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 2, v19.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.h, 1, v29.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.h, v1.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v16.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v6.h, 3, v5.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v31.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v3.h, v7.h, v3.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 2, v8.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.h, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v29.l
; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v9.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v8.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v11.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 2, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, 15, v3.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v5.h, v6.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v28.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v6.h, v14.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.l, 1, v13.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.l, v17.h, 15, v17.l bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.l, v16.l, v18.l, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v19.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v18.l, v28.l, v19.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v2.h, v4.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_and_b16 v7.l, v8.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v11.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.l, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v6.l, v5.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v10.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 1, v9.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 2, v6.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.h, 1, v13.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v1.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, 3, v3.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v5.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v6.h, v5.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v12.l, v7.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v18.l, v17.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.l, v4.l, v5.l, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v7.l, 3, v6.l bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v5.h, v7.h, v5.h
+; GFX1250-TRUE16-NEXT: v_or_b16 v6.l, v8.l, v6.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v6.h, v12.l, v8.h, 1 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v3.h, 15, v4.l bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v5.l, v4.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v16.h, 4, v16.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v17.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v4.l, v4.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v5.l, 15, v5.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.l, v6.h, v6.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 4, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 12, v2.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v16.l, v16.h, 15 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v17.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 4, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 12, v4.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.h, v1.h, 15 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v3.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 15 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v2.l, v2.h, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v2.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.h, v1.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v0.h, 0xff bitop3:0xec
-; GFX1250-TRUE16-NEXT: global_store_b32 v[0:1], v1, off
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: global_store_b32 v[0:1], v0, off
; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: amdgpu_cs_v32i1:
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
index f8b1448a612e0..afcc3dec134c9 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -2576,9 +2576,9 @@ define i1 @test131(i16 %arg1, i32 %arg2) {
; GFX11-TRUE16-LABEL: test131:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_gt_u32_e32 vcc_lo, 10, v1
-; GFX11-TRUE16-NEXT: v_cmp_gt_u16_e64 s0, 10, v0.l
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_u16_e32 vcc_lo, 10, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_u32_e64 s0, 10, v1
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 6bc5844c40324..c0053b0f1ef32 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -299,23 +299,14 @@ define <2 x float> @v_uitofp_v2i8_to_v2f32(i16 %arg0) nounwind {
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_uitofp_v2i8_to_v2f32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v0, v1
-; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte1_e32 v1, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_uitofp_v2i8_to_v2f32:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_uitofp_v2i8_to_v2f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
+; GFX11-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val = bitcast i16 %arg0 to <2 x i8>
%cvt = uitofp <2 x i8> %val to <2 x float>
ret <2 x float> %cvt
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index c9de2001e024a..98fea4091c58f 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -2203,11 +2203,11 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4200
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc800, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4200
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0xc800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test6:
@@ -2313,11 +2313,11 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4800, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc400
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0x4800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test7:
@@ -2401,9 +2401,9 @@ define half @fmul_select_f16_test8(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test8:
@@ -2837,21 +2837,22 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3024,21 +3025,22 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test2:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3f00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3826,21 +3828,22 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test5:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4100
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4015,21 +4018,22 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4040
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4203,21 +4207,22 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc080
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4573,21 +4578,22 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test9:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc200
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc180, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4762,21 +4768,22 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xdb80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xe000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4951,21 +4958,22 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4c00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3480, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index 60f7125f9b628..88f6db24e28c6 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -258,10 +258,7 @@ define float @divergent_vec_f16_neg_0(half %a) {
; GFX11-TRUE16-LABEL: divergent_vec_f16_neg_0:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x8000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x8000
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: divergent_vec_f16_neg_0:
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 028ec031941d3..275c09bfa493f 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -3205,7 +3205,7 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX11-GISEL-NEXT: scratch_store_b32 off, v1, s33
; GFX11-GISEL-NEXT: scratch_store_b32 off, v2, s33 offset:4
; GFX11-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-GISEL-NEXT: s_add_i32 s32, s32, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: s_mov_b32 s0, s32
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index 99a0241f2d4a6..3819863d402ce 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -1247,10 +1247,9 @@ define <2 x bfloat> @v_test_canonicalize_reg_k_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, 0x4000
-; REAL16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <2 x bfloat> poison, bfloat %val, i32 0
%vec1 = insertelement <2 x bfloat> %vec0, bfloat 2.0, i32 1
@@ -1273,10 +1272,10 @@ define <2 x bfloat> @v_test_canonicalize_k_reg_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0x4000
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.l, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <2 x bfloat> poison, bfloat 2.0, i32 0
%vec1 = insertelement <2 x bfloat> %vec0, bfloat %val, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index d55886e13e998..1d237068dab04 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -3040,11 +3040,11 @@ define <4 x half> @v_test_canonicalize_reg_undef_reg_reg_v4f16(half %val0, half
; GFX11-TRUE16-LABEL: v_test_canonicalize_reg_undef_reg_reg_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_test_canonicalize_reg_undef_reg_reg_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 9db4987fc8210..5e76d173bb669 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -266,14 +266,23 @@ define i16 @s_test_copysign_f16_10_mag(half inreg %sign) {
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: s_test_copysign_f16_10_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_10_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_10_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.copysign.f16(half 10.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -983,101 +992,52 @@ define half @v_copysign_out_f16_mag_f64_sign_f16(double %mag, half %sign) {
; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v1, 20, 11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
-; GFX11-TRUE16-NEXT: v_med3_i32 v3, v4, 0, 13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x1000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v3, v5
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v1, 12, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 7, v3
-; GFX11-TRUE16-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v1, 20, 11
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
-; GFX11-FAKE16-NEXT: v_med3_i32 v3, v4, 0, 13
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x1000, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v3, v5
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v1, 12, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 7, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX11-FAKE16-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0x7e00 :: v_dual_add_nc_u32 v3, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v4, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-NEXT: v_bfe_u32 v1, v1, 20, 11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
+; GFX11-NEXT: v_med3_i32 v3, v4, 0, 13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v4, 0x1000, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v3, v5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_lshl_or_b32 v4, v1, 12, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v4, 7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 2, v3
+; GFX11-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_dual_mov_b32 v4, 0x7e00 :: v_dual_add_nc_u32 v3, v3, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v4, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%mag.trunc = fptrunc double %mag to half
%result = call half @llvm.copysign.f16(half %mag.trunc, half %sign)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
index d3a526dfbe425..50af631d58bb4 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
@@ -184,11 +184,7 @@ define <2 x bfloat> @v_rsq_bf16_multi_use(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v1.h, v1.l
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v0.h, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: v_rsq_bf16_multi_use:
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
index aeca505f7c504..acd7b41cc7196 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
@@ -1931,9 +1931,9 @@ define half @v_fdiv_f16_arcp(half %x, half %y) {
; GFX11-TRUE16-LABEL: v_fdiv_f16_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_fdiv_f16_arcp:
@@ -1982,9 +1982,9 @@ define half @v_fdiv_f16_afn_nsz(half %x, half %y) {
; GFX11-TRUE16-LABEL: v_fdiv_f16_afn_nsz:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_fdiv_f16_afn_nsz:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 0a6dbff0e55f1..8c86df6a4ff59 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11952,58 +11952,114 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12048,54 +12104,106 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12295,73 +12403,131 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v3, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
@@ -12396,56 +12562,110 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12651,61 +12871,119 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12753,56 +13031,110 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -13708,32 +14040,33 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13741,7 +14074,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -13826,40 +14159,41 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -14348,56 +14682,111 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14441,52 +14830,103 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14681,62 +15121,121 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14783,56 +15282,110 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 91d75d7077259..d4f94afe193a6 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -6427,32 +6427,30 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6460,7 +6458,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6558,40 +6556,38 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6833,32 +6829,30 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6866,7 +6860,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6965,40 +6959,38 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7621,30 +7613,28 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7749,31 +7739,29 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8017,30 +8005,28 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8146,31 +8132,29 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9007,33 +8991,31 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9041,7 +9023,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9140,40 +9122,38 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9420,31 +9400,29 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9550,31 +9528,29 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9810,58 +9786,114 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9906,89 +9938,141 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v5, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: flat_load_dword v5, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v4, v4
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v6, v5
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
@@ -10154,61 +10238,119 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10255,56 +10397,110 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10511,61 +10707,119 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10613,56 +10867,110 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10869,56 +11177,111 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10962,52 +11325,103 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11905,32 +12319,33 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11938,7 +12353,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12023,40 +12438,41 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12547,62 +12963,121 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12649,56 +13124,110 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index f5564f14eee24..def9fe8c0f6b4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -6427,32 +6427,30 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6460,7 +6458,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6558,40 +6556,38 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6833,32 +6829,30 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6866,7 +6860,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6965,40 +6959,38 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7621,30 +7613,28 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7749,31 +7739,29 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8017,30 +8005,28 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8146,31 +8132,29 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9007,33 +8991,31 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9041,7 +9023,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9140,40 +9122,38 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9420,31 +9400,29 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9550,31 +9528,29 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9810,58 +9786,114 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9906,89 +9938,141 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v5, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: flat_load_dword v5, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v4, v4
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v6, v5
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
@@ -10154,61 +10238,119 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10255,56 +10397,110 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10511,61 +10707,119 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10613,56 +10867,110 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10869,56 +11177,111 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10962,52 +11325,103 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11905,32 +12319,33 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11938,7 +12353,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12023,40 +12438,41 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12547,62 +12963,121 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12649,56 +13124,110 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 2c64aaf801f1f..45c72e7f94a37 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9383,58 +9383,114 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9479,54 +9535,106 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9726,61 +9834,119 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9827,56 +9993,110 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10082,61 +10302,119 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10184,56 +10462,110 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10439,56 +10771,111 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
}
define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10532,52 +10919,103 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -11472,32 +11910,33 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11505,7 +11944,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -11590,40 +12029,41 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12112,62 +12552,121 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
}
define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12214,56 +12713,110 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
index 40c256b106a2a..75b939778d0d7 100644
--- a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
@@ -1268,27 +1268,27 @@ define <8 x i1> @vector_f32_ole() {
; SDAG-GFX1150-LABEL: vector_f32_ole:
; SDAG-GFX1150: ; %bb.0: ; %entry
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; SDAG-GFX1150-NEXT: s_clause 0x1
-; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[0:1], off
-; SDAG-GFX1150-NEXT: global_load_b128 v[4:7], v[4:5], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[4:7], v[0:1], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[2:3], off
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(1)
-; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v3
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
-; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v4
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v2
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v6
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v5
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v4
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v3
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v1
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v0
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v6
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v2
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v7
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_setpc_b64 s[30:31]
@@ -1314,24 +1314,28 @@ define <8 x i1> @vector_f32_ole() {
; GISEL-GFX1150-NEXT: v_readfirstlane_b32 s7, v7
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s4, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_cselect_b32 s4, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s5, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v4.l, s4
; GISEL-GFX1150-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s6, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1150-NEXT: s_cselect_b32 s6, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s7, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v6.l, s6
; GISEL-GFX1150-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v7.l, s7
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ole:
@@ -1341,34 +1345,34 @@ define <8 x i1> @vector_f32_ole() {
; SDAG-GFX1200-NEXT: s_wait_samplecnt 0x0
; SDAG-GFX1200-NEXT: s_wait_bvhcnt 0x0
; SDAG-GFX1200-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; SDAG-GFX1200-NEXT: s_clause 0x1
-; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[0:1], off
-; SDAG-GFX1200-NEXT: global_load_b128 v[4:7], v[4:5], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[4:7], v[0:1], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[2:3], off
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x1
-; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v3
+; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v6
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v5
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v4
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v2
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v5
+; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v1
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v0
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v6
+; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_ge_f32_e32 vcc_lo, 0, v7
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
@@ -1399,27 +1403,35 @@ define <8 x i1> @vector_f32_ole() {
; GISEL-GFX1200-NEXT: v_readfirstlane_b32 s7, v7
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s4, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_cselect_b32 s4, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s5, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v4.l, s4
; GISEL-GFX1200-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s6, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1200-NEXT: s_cselect_b32 s6, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s7, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v6.l, s6
; GISEL-GFX1200-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v7.l, s7
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <8 x float>, ptr addrspace(1) null, align 32
@@ -1434,12 +1446,12 @@ define <4 x i1> @vector_f32_ogt() {
; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
-; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v1
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v0
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v0
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v1
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_setpc_b64 s[30:31]
@@ -1457,14 +1469,16 @@ define <4 x i1> @vector_f32_ogt() {
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s0, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ogt:
@@ -1477,15 +1491,15 @@ define <4 x i1> @vector_f32_ogt() {
; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v1
+; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v0
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v2
+; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v1
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
@@ -1508,15 +1522,19 @@ define <4 x i1> @vector_f32_ogt() {
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s0, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <4 x float>, ptr addrspace(1) null, align 16
@@ -1528,97 +1546,95 @@ define <32 x i1> @vector_f32_ueq() {
; SDAG-GFX1150-LABEL: vector_f32_ueq:
; SDAG-GFX1150: ; %bb.0: ; %entry
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 48 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v4, 64 :: v_dual_mov_b32 v5, 0
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v8, 32 :: v_dual_mov_b32 v9, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 0x70 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v2, 0x60 :: v_dual_mov_b32 v3, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v4, 0x50 :: v_dual_mov_b32 v5, 0
; SDAG-GFX1150-NEXT: s_clause 0x1
-; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[0:1], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[28:31], v[0:1], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[2:3], off
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v8, 64 :: v_dual_mov_b32 v9, 0
; SDAG-GFX1150-NEXT: global_load_b128 v[4:7], v[4:5], off
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v12, 0x50 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1150-NEXT: s_clause 0x1
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v12, 48 :: v_dual_mov_b32 v13, 0
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v16, 32 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1150-NEXT: s_clause 0x2
; SDAG-GFX1150-NEXT: global_load_b128 v[8:11], v[8:9], off
-; SDAG-GFX1150-NEXT: global_load_b128 v[20:23], v[12:13], off
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v14, 0x60 :: v_dual_mov_b32 v15, 0
-; SDAG-GFX1150-NEXT: s_clause 0x1
-; SDAG-GFX1150-NEXT: global_load_b128 v[27:30], v[12:13], off
-; SDAG-GFX1150-NEXT: global_load_b128 v[31:34], v[14:15], off
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1150-NEXT: v_dual_mov_b32 v14, 0x70 :: v_dual_mov_b32 v15, 0
-; SDAG-GFX1150-NEXT: s_clause 0x1
-; SDAG-GFX1150-NEXT: global_load_b128 v[35:38], v[12:13], off
-; SDAG-GFX1150-NEXT: global_load_b128 v[48:51], v[14:15], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[12:15], v[12:13], off
+; SDAG-GFX1150-NEXT: global_load_b128 v[32:35], v[16:17], off
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v16, 16 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1150-NEXT: global_load_b128 v[36:39], v[16:17], off
+; SDAG-GFX1150-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1150-NEXT: global_load_b128 v[48:51], v[16:17], off
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(7)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v3
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v30
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v30, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v29
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v29, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v28
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v28, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(6)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v4
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v3
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v27, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v2
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v5
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v26, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v1
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v6
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v25, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v7
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(5)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v11
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(4)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v20
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v7
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v23, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v6
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v22, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v5
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v4
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(4)
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v11
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v10
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v21
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v9
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v22
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v22, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v8
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v23
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v23, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(3)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v30
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v15
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v14
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v13
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v12
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(2)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v31
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v29
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v32
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v25, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v28
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v33
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v26, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v27
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v35
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v34
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v27, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v33
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v32
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(1)
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v39
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v38
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v48
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v28, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v37
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v49
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v29, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v36
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v51
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v49
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v35
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v48
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v50
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v30, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v51
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v31
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v31, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
@@ -1702,68 +1718,84 @@ define <32 x i1> @vector_f32_ueq() {
; GISEL-GFX1150-NEXT: v_readfirstlane_b32 s41, v31
; GISEL-GFX1150-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s9, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s10, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v4.l, s4
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1150-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s11, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v6.l, s6
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v7.l, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v8.l, s8
; GISEL-GFX1150-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s12, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v9.l, s9
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v10.l, s10
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v11.l, s11
; GISEL-GFX1150-NEXT: s_cselect_b32 s12, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s13, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v12.l, s12
; GISEL-GFX1150-NEXT: s_cselect_b32 s13, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s14, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v13.l, s13
; GISEL-GFX1150-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s15, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v14.l, s14
; GISEL-GFX1150-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s16, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v15.l, s15
; GISEL-GFX1150-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s17, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v16.l, s16
; GISEL-GFX1150-NEXT: s_cselect_b32 s17, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s18, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v17.l, s17
; GISEL-GFX1150-NEXT: s_cselect_b32 s18, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s19, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v18.l, s18
; GISEL-GFX1150-NEXT: s_cselect_b32 s19, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s20, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v19.l, s19
; GISEL-GFX1150-NEXT: s_cselect_b32 s20, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s21, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v20.l, s20
; GISEL-GFX1150-NEXT: s_cselect_b32 s21, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s22, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v21.l, s21
; GISEL-GFX1150-NEXT: s_cselect_b32 s22, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s23, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v22.l, s22
; GISEL-GFX1150-NEXT: s_cselect_b32 s23, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s24, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v23.l, s23
; GISEL-GFX1150-NEXT: s_cselect_b32 s24, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s25, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v24.l, s24
; GISEL-GFX1150-NEXT: s_cselect_b32 s25, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s26, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v25.l, s25
; GISEL-GFX1150-NEXT: s_cselect_b32 s26, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s27, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v26.l, s26
; GISEL-GFX1150-NEXT: s_cselect_b32 s27, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s28, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v27.l, s27
; GISEL-GFX1150-NEXT: s_cselect_b32 s28, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s29, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v28.l, s28
; GISEL-GFX1150-NEXT: s_cselect_b32 s29, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s40, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v29.l, s29
; GISEL-GFX1150-NEXT: s_cselect_b32 s40, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s41, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v30.l, s40
; GISEL-GFX1150-NEXT: s_cselect_b32 s41, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v30, s40 :: v_dual_mov_b32 v31, s41
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v31.l, s41
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ueq:
@@ -1773,128 +1805,126 @@ define <32 x i1> @vector_f32_ueq() {
; SDAG-GFX1200-NEXT: s_wait_samplecnt 0x0
; SDAG-GFX1200-NEXT: s_wait_bvhcnt 0x0
; SDAG-GFX1200-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 48 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v4, 64 :: v_dual_mov_b32 v5, 0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v8, 32 :: v_dual_mov_b32 v9, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 0x70 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v2, 0x60 :: v_dual_mov_b32 v3, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v4, 0x50 :: v_dual_mov_b32 v5, 0
; SDAG-GFX1200-NEXT: s_clause 0x1
-; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[0:1], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[28:31], v[0:1], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[2:3], off
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v8, 64 :: v_dual_mov_b32 v9, 0
; SDAG-GFX1200-NEXT: global_load_b128 v[4:7], v[4:5], off
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v12, 0x50 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1200-NEXT: s_clause 0x1
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v12, 48 :: v_dual_mov_b32 v13, 0
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v16, 32 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1200-NEXT: s_clause 0x2
; SDAG-GFX1200-NEXT: global_load_b128 v[8:11], v[8:9], off
-; SDAG-GFX1200-NEXT: global_load_b128 v[20:23], v[12:13], off
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v14, 0x60 :: v_dual_mov_b32 v15, 0
-; SDAG-GFX1200-NEXT: s_clause 0x1
-; SDAG-GFX1200-NEXT: global_load_b128 v[27:30], v[12:13], off
-; SDAG-GFX1200-NEXT: global_load_b128 v[31:34], v[14:15], off
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
-; SDAG-GFX1200-NEXT: v_dual_mov_b32 v14, 0x70 :: v_dual_mov_b32 v15, 0
-; SDAG-GFX1200-NEXT: s_clause 0x1
-; SDAG-GFX1200-NEXT: global_load_b128 v[35:38], v[12:13], off
-; SDAG-GFX1200-NEXT: global_load_b128 v[48:51], v[14:15], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[12:15], v[12:13], off
+; SDAG-GFX1200-NEXT: global_load_b128 v[32:35], v[16:17], off
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v16, 16 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1200-NEXT: global_load_b128 v[36:39], v[16:17], off
+; SDAG-GFX1200-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1200-NEXT: global_load_b128 v[48:51], v[16:17], off
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x7
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v3
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v30
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v30, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v29
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v29, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v28
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v28, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x6
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v4
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v27, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v5
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v26, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v1
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v6
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v25, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x5
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v7
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x5
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v11
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v23, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v6
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x4
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v20
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v22, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v5
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v4
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v10
+; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x4
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v11
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v21
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v10
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v9
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v22
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v22, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v8
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v23
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v23, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x3
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v30
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v15
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x2
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v31
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v14
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v29
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v13
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v32
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v12
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v25, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v28
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x2
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v35
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v33
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v34
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v26, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v27
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v33
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v34
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v32
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v27, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x1
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v38
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v39
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v48
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v38
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v28, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v37
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v49
-; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v29, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v36
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v51
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v49
+; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v35
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v48
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v50
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v30, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v51
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v31
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v31, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: s_setpc_b64 s[30:31]
@@ -1984,79 +2014,104 @@ define <32 x i1> @vector_f32_ueq() {
; GISEL-GFX1200-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s9, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s10, 0
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v4.l, s4
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1200-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s11, 0
-; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v6.l, s6
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v7.l, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v8.l, s8
; GISEL-GFX1200-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s12, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v9.l, s9
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v10.l, s10
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v11.l, s11
; GISEL-GFX1200-NEXT: s_cselect_b32 s12, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s13, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v12.l, s12
; GISEL-GFX1200-NEXT: s_cselect_b32 s13, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s14, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v13.l, s13
; GISEL-GFX1200-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s15, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v14.l, s14
; GISEL-GFX1200-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s16, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v15.l, s15
; GISEL-GFX1200-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s17, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v16.l, s16
; GISEL-GFX1200-NEXT: s_cselect_b32 s17, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s18, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v17.l, s17
; GISEL-GFX1200-NEXT: s_cselect_b32 s18, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s19, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v18.l, s18
; GISEL-GFX1200-NEXT: s_cselect_b32 s19, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s20, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v19.l, s19
; GISEL-GFX1200-NEXT: s_cselect_b32 s20, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s21, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v20.l, s20
; GISEL-GFX1200-NEXT: s_cselect_b32 s21, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s22, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v21.l, s21
; GISEL-GFX1200-NEXT: s_cselect_b32 s22, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s23, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v22.l, s22
; GISEL-GFX1200-NEXT: s_cselect_b32 s23, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s24, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v23.l, s23
; GISEL-GFX1200-NEXT: s_cselect_b32 s24, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s25, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v24.l, s24
; GISEL-GFX1200-NEXT: s_cselect_b32 s25, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s26, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v25.l, s25
; GISEL-GFX1200-NEXT: s_cselect_b32 s26, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s27, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v26.l, s26
; GISEL-GFX1200-NEXT: s_cselect_b32 s27, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s28, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v27.l, s27
; GISEL-GFX1200-NEXT: s_cselect_b32 s28, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s29, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v28.l, s28
; GISEL-GFX1200-NEXT: s_cselect_b32 s29, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s40, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v29.l, s29
; GISEL-GFX1200-NEXT: s_cselect_b32 s40, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s41, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v30.l, s40
; GISEL-GFX1200-NEXT: s_cselect_b32 s41, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v30, s40 :: v_dual_mov_b32 v31, s41
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v31.l, s41
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <32 x float>, ptr addrspace(1) null, align 128
@@ -2071,12 +2126,12 @@ define <4 x i1> @vector_f32_ugt() {
; SDAG-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1150-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-GFX1150-NEXT: s_waitcnt vmcnt(0)
-; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v1
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v0
-; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v0
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v1
+; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1150-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
; SDAG-GFX1150-NEXT: s_setpc_b64 s[30:31]
@@ -2094,14 +2149,16 @@ define <4 x i1> @vector_f32_ugt() {
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s0, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ugt:
@@ -2114,15 +2171,15 @@ define <4 x i1> @vector_f32_ugt() {
; SDAG-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1200-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-GFX1200-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v1
+; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v2
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v0
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v2
+; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v1
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; SDAG-GFX1200-NEXT: v_cmp_nge_f32_e32 vcc_lo, 0, v3
; SDAG-GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; SDAG-GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
@@ -2145,15 +2202,19 @@ define <4 x i1> @vector_f32_ugt() {
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s0, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV1 = load <4 x float>, ptr addrspace(1) null, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
index cd48c0dc3bd5a..fe361429ef547 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
@@ -1084,10 +1084,11 @@ define <4 x i8> @to_fp8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX1250-TRUE16-NEXT: v_dual_lshrrev_b32 v1, 8, v0 :: v_dual_lshrrev_b32 v3, 24, v3
+; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v3, 16, v2 :: v_dual_lshrrev_b32 v1, 8, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_fp8_v4f16:
@@ -1349,10 +1350,11 @@ define <4 x i8> @to_bf8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v0.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX1250-TRUE16-NEXT: v_dual_lshrrev_b32 v1, 8, v0 :: v_dual_lshrrev_b32 v3, 24, v3
+; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v3, 16, v2 :: v_dual_lshrrev_b32 v1, 8, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_bf8_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index fe0b62e662f61..cc56e0f35f668 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1592,90 +1592,93 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX1200-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1200-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v2, v5
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
-; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX1200-NEXT: v_xor_b16 v1.l, v6.l, -1
-; GFX1200-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX1200-NEXT: v_min_u16 v1.l, v1.l, 15
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc_lo
-; GFX1200-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v1, v5
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
+; GFX1200-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_xor_b16 v0.h, v6.l, -1
+; GFX1200-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX1200-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_min_u16 v0.h, v0.h, 15
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc_lo
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp
; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, v1.h, 1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshlrev_b16 v1.h, v1.l, 1
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v2.h, 0x7f, v3.l
-; GFX1200-NEXT: v_and_b16 v3.l, v3.l, 7
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, -1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v3.h, 0x80, v2.h
-; GFX1200-NEXT: v_lshrrev_b16 v2.h, 4, v2.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v2.l, v3.h, v2.l
-; GFX1200-NEXT: v_and_b16 v3.l, v2.h, 1
+; GFX1200-NEXT: v_and_b16 v2.l, 0x7f, v3.l
+; GFX1200-NEXT: v_and_b16 v2.h, v3.l, 7
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, -1
+; GFX1200-NEXT: v_or_b16 v3.l, 0x80, v2.l
+; GFX1200-NEXT: v_lshrrev_b16 v2.l, 4, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v2.h, v2.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v1.l, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, v0.h, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v1.l, v1.l, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v2.l, v4.l, v3.l
-; GFX1200-NEXT: v_and_b16 v3.l, v4.h, 1
+; GFX1200-NEXT: v_or_b16 v1.h, v4.l, v2.h
+; GFX1200-NEXT: v_and_b16 v2.h, v3.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
-; GFX1200-NEXT: v_and_b16 v2.l, v7.l, v2.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h
+; GFX1200-NEXT: v_and_b16 v1.h, v7.l, v1.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.h, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v2.l, v1.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v3.l, v4.l, v3.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
+; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_and_b16 v1.l, v1.h, v3.l
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0
; GFX1200-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_add_nc_u16 v0.h, v4.h, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo
+; GFX1200-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_add_nc_u16 v0.h, v3.h, v0.h
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v1.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, 6
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 8, vcc_lo
; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v1.h, v0.l, v1.h
+; GFX1200-NEXT: v_or_b16 v2.l, v0.l, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v2.h, v0.l, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v2.l
+; GFX1200-NEXT: v_or_b16 v0.h, v2.l, v0.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v1.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v5
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
@@ -1683,73 +1686,74 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0x80
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x80
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
-; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_xor_b16 v1.l, v7.l, -1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
+; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v6
+; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, v7.l, -1
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 15
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, 0x7f, v2.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v2.l, 7
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.h, 4, v2.h
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
+; GFX1250-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 15
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v1.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v1.l, 7
+; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v0.h, 1 clamp
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, v1.h, 1
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.l, 4, v2.l
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, v1.h, 1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v3.h, -1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v2.l, v4.l, 0x7f bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.l, 3, v2.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, v4.h, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v2.h, v3.l, 0x80 bitop3:0xc8
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.l, v1.l, v3.h
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v5.l, v4.l bitop3:0xe0
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, -1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v1.l, v3.h, 0x7f bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, 3, v1.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v3.l, 1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.h, v2.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v2.h, 0x80 bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.h, v0.h, v3.l
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v5.l, v3.h bitop3:0xe0
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.h, 1
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, v1.l
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
-; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.h, v4.l, v2.h bitop3:0xe0
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, 0, s0
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s0
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v3.l, v2.l bitop3:0xe0
; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, v4.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, v1.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, 6
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.h, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v0.h
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v1.h
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, vcc_lo
-; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v1.h, v2.l bitop3:0xfe
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.h bitop3:0xfe
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.l, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.l bitop3:0xfe
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v0.h, v2.h bitop3:0xfe
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
index 9f8bbef48d20f..4568866830a0c 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
@@ -786,26 +786,26 @@ define <5 x i8> @to_fp8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l
; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63
; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h
@@ -814,172 +814,168 @@ define <5 x i8> @to_fp8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l
; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l
+; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h
; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v5.l
-; GFX1200-NEXT: v_or_b16 v5.l, 0x400, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v7.h, -1
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v4.h, v5.h, v4.h
-; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h
+; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v7.h, v9.l, 1
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l
; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v2.l
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.h, s1
-; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
-; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
+; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15
+; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, v2.h, 1 clamp
-; GFX1200-NEXT: v_and_b16 v1.h, v5.l, v1.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
-; GFX1200-NEXT: v_and_b16 v3.h, 0x3ff, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
-; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l
; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v2.h, v7.h
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1
; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h
; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
+; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h
; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l
; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2
-; GFX1200-NEXT: v_and_b16 v3.h, v4.h, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l
+; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h
; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l
; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v3.h, s1
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l
; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v10.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l
-; GFX1200-NEXT: v_and_b16 v3.h, 0x80, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0
; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v2.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
-; GFX1200-NEXT: v_or_b16 v0.h, v3.h, v5.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h
; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6
; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v6.l, v3.h, v6.l
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h
; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0
-; GFX1200-NEXT: v_or_b16 v2.h, v6.h, v2.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
+; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v2.h, s1
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1
; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
-; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_or_b32_e32 v4, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
@@ -1558,26 +1554,26 @@ define <6 x i8> @to_fp8_v6f16(<6 x half> %x) {
; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l
; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63
; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h
@@ -1586,174 +1582,170 @@ define <6 x i8> @to_fp8_v6f16(<6 x half> %x) {
; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l
; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l
+; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h
; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v5.l
-; GFX1200-NEXT: v_or_b16 v5.l, 0x400, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v7.h, -1
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v4.h, v5.h, v4.h
-; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h
+; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v7.h, v9.l, 1
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l
; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v2.l
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.h, s1
-; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
-; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
+; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15
+; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, v2.h, 1 clamp
-; GFX1200-NEXT: v_and_b16 v1.h, v5.l, v1.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
-; GFX1200-NEXT: v_and_b16 v3.h, 0x3ff, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
-; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l
; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v2.h, v7.h
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1
; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h
; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
+; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h
; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l
; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2
-; GFX1200-NEXT: v_and_b16 v3.h, v4.h, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l
+; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h
; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l
; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v3.h, s1
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l
; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v10.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l
-; GFX1200-NEXT: v_and_b16 v3.h, 0x80, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0
; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v2.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
-; GFX1200-NEXT: v_or_b16 v0.h, v3.h, v5.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h
; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6
; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v6.l, v3.h, v6.l
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h
; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0
-; GFX1200-NEXT: v_or_b16 v2.h, v6.h, v2.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
+; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v2.h, s1
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1
; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
-; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_or_b32_e32 v5, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_lshrrev_b64 v[7:8], 24, v[5:6]
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
@@ -2729,28 +2721,29 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v6.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l
; GFX1200-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.h, s1
; GFX1200-NEXT: v_and_b16 v1.h, 0xff, v4.h
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l
+; GFX1200-NEXT: v_or_b16 v4.h, v4.l, v3.h
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v4.l, v3.h
+; GFX1200-NEXT: v_or_b16 v4.l, v1.h, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_or_b16 v6.l, v1.h, v2.h
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX1200-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX1200-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX1200-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_fp8_v7f16:
@@ -2758,20 +2751,21 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v4.l, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.h, v3
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.l, v2
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.h, v3
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v2
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[6:7]
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[2:3]
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.h
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_fp8_v7f16:
@@ -3443,6 +3437,7 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s1
; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.l
+; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
@@ -3458,7 +3453,7 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1
; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.l
-; GFX1200-NEXT: v_cndmask_b16 v8.h, v4.l, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, vcc_lo
; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15
; GFX1200-NEXT: v_lshlrev_b16 v6.h, 7, v7.l
; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, 14
@@ -3466,185 +3461,184 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, 4, s0
; GFX1200-NEXT: v_sub_nc_u16 v6.l, v4.h, 1 clamp
-; GFX1200-NEXT: v_or_b16 v4.l, v2.h, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v1.h
; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7e, v3.l, s1
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
-; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v7.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7e, v3.l, s1
; GFX1200-NEXT: v_or_b16 v7.l, v6.h, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v8.h
+; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v7.h
; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
+; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v4.l
; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v1.h
-; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v8.h
-; GFX1200-NEXT: v_and_b16 v3.l, v9.l, v8.l
; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v1.h
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, v4.h, v9.l
+; GFX1200-NEXT: v_and_b16 v3.h, v8.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, v4.h, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v8.h
; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v1.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v3.h
+; GFX1200-NEXT: v_and_b16 v1.h, v4.l, 1
; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1
-; GFX1200-NEXT: v_and_b16 v1.h, v3.h, 1
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v12.l
-; GFX1200-NEXT: v_and_b16 v7.l, 0x7f, v5.h
+; GFX1200-NEXT: v_and_b16 v3.h, 0x7f, v5.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s2
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s2
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
-; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v1.h, v3.l, v1.h
-; GFX1200-NEXT: v_lshrrev_b16 v3.l, v6.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v7.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, 8, v7.h
; GFX1200-NEXT: v_sub_nc_u16 v7.h, -5, v2.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v8.l
-; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h
-; GFX1200-NEXT: v_and_b16 v4.h, v6.l, 1
-; GFX1200-NEXT: v_min_u16 v3.l, v7.h, 15
+; GFX1200-NEXT: v_and_b16 v1.h, v6.l, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v4.h, v7.l, 1
+; GFX1200-NEXT: v_min_u16 v3.h, v7.h, 15
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v12.l
-; GFX1200-NEXT: v_or_b16 v7.h, 0x7c, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v3.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v8.l
+; GFX1200-NEXT: v_or_b16 v7.h, 0x7c, v6.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h
+; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v4.l, v1.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s0
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v3.h, v1.h
-; GFX1200-NEXT: v_and_b16 v3.h, v5.h, v4.h
; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v7.l
+; GFX1200-NEXT: v_and_b16 v4.l, v5.h, v4.h
+; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v6.l
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.l, v3.h
; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, -1
; GFX1200-NEXT: v_and_b16 v9.l, 0x7f, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v3.l, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.l, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v3.h, v4.h
; GFX1200-NEXT: v_and_b16 v5.h, v4.h, v5.h
; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
; GFX1200-NEXT: v_and_b16 v9.l, v6.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v1.h
; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v4.l
; GFX1200-NEXT: v_and_b16 v5.h, v7.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.h, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s2
; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.h, v4.h
; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v0.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
; GFX1200-NEXT: v_and_b16 v6.h, v8.l, v9.l
; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, s0
; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v11.l
-; GFX1200-NEXT: v_and_b16 v3.l, 0x80, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h
; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_and_b16 v3.h, 0x80, v8.h
; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14
-; GFX1200-NEXT: v_or_b16 v6.h, v3.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s1
; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v5.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v3.h
+; GFX1200-NEXT: v_or_b16 v6.h, v3.h, v8.l
; GFX1200-NEXT: v_lshlrev_b16 v5.h, 2, v5.l
-; GFX1200-NEXT: v_or_b16 v1.h, v6.h, v1.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v4.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX1200-NEXT: v_or_b16 v1.h, v6.h, v1.h
+; GFX1200-NEXT: v_or_b16 v5.h, v3.h, v5.h
; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v5.l
-; GFX1200-NEXT: v_or_b16 v5.h, v3.l, v5.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.l
; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v4.h
; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.l
+; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v5.l
-; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v6.l, 0, s0
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h
; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v8.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v6.l, 0, s0
; GFX1200-NEXT: s_and_b32 s0, s3, s2
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
; GFX1200-NEXT: v_add_nc_u16 v0.h, v2.l, 14
; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 4, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v3.h, 7, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v4.l, 7, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s3, s4, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l
; GFX1200-NEXT: v_lshlrev_b16 v6.h, 2, v0.h
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h
-; GFX1200-NEXT: v_or_b16 v2.l, v3.h, v2.l
+; GFX1200-NEXT: v_or_b16 v2.l, v4.l, v2.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l
; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v0.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v0.h
-; GFX1200-NEXT: v_or_b16 v6.l, v3.h, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s5
; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s5
; GFX1200-NEXT: s_and_b32 s0, s1, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l
; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v5.l
+; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v3.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s0, s2, s0
-; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v3.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l
; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v2.l, 0x7c, v3.h
+; GFX1200-NEXT: v_or_b16 v2.l, 0x7c, v4.l
; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s3
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s1
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v2.l, s0
; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v3.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s0
; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s2
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v4, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v2.l, s0
; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7e7e0000, v3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v2.l, v2.h, v0.h
+; GFX1200-NEXT: v_or_b32_e32 v4, 0x7e7e0000, v4
+; GFX1200-NEXT: v_or_b16 v2.l, v3.l, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7e, v0.l, s0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_bf8_v5f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index aa6f54af57d16..688aaa7ff4125 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -407,13 +407,12 @@ define i32 @test_D139469_f16(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: test_D139469_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x291e
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v1.l, 0x211e
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.h, 0x291e, v1.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_f16_e32 v0.l, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x291e
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v1.l, 0x291e, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v0.h, 0x211e
+; GFX11-SDAG-TRUE16-NEXT: v_min_f16_e32 v0.l, v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -465,13 +464,12 @@ define i32 @test_D139469_f16(half %arg) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x291e
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v1.l, 0x211e
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.h, 0x291e, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x291e
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v1.l, 0x291e, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v0.h, 0x211e
+; GFX12-SDAG-TRUE16-NEXT: v_min_num_f16_e32 v0.l, v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index 98a6c22301193..b7f72c58c5264 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1909,47 +1909,49 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2002,17 +2004,19 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -2021,35 +2025,35 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 6e2c47ac62c24..7959e40da4a16 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -485,12 +485,26 @@ define amdgpu_ps half @test_fmaximum_f16_ss(half inreg %a, half inreg %b) {
; GFX1170-GISEL-FAKE16-NEXT: v_maximum_f16 v0, s0, s1
; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: test_fmaximum_f16_ss:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmaximum_f16_ss:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: test_fmaximum_f16_ss:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: test_fmaximum_f16_ss:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%val = call half @llvm.maximum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
index c534f89fd3d7e..3bc8dd8c6a7ac 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
@@ -2416,9 +2416,9 @@ define half @v_fmaximum3_f16_const1_const2(half %a) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX12-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_fmaximum3_f16_const1_const2:
@@ -2436,9 +2436,9 @@ define half @v_fmaximum3_f16_const1_const2(half %a) {
; GFX1170-TRUE16-LABEL: v_fmaximum3_f16_const1_const2:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX1170-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_fmaximum3_f16_const1_const2:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 9851a14074d18..3ddc0a6d77980 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -2086,12 +2086,19 @@ define amdgpu_ps half @test_fmax_f16_s_ieee_off(half inreg %a, half inreg %b) #0
; GFX9-NEXT: v_max_f16_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: test_fmax_f16_s_ieee_off:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_max_num_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmax_f16_s_ieee_off:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_max_num_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fmax_f16_s_ieee_off:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_max_num_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.maxnum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
index 0b1b5a6359547..785cb88c1a379 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,16 +69,17 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 {
; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index 08b7279c6221b..7676c58005d90 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1911,47 +1911,49 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2004,17 +2006,19 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -2023,35 +2027,35 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 2156fc6f2397c..ceb2d16289787 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -485,12 +485,26 @@ define amdgpu_ps half @test_fminimum_f16_ss(half inreg %a, half inreg %b) {
; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v0, s0, s1
; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: test_fminimum_f16_ss:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_minimum_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fminimum_f16_ss:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: test_fminimum_f16_ss:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: test_fminimum_f16_ss:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%val = call half @llvm.minimum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
index 2aac73fae170d..9a522f5239752 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
@@ -2416,9 +2416,9 @@ define half @v_fminimum3_f16_const1_const2(half %a) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX12-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_fminimum3_f16_const1_const2:
@@ -2436,9 +2436,9 @@ define half @v_fminimum3_f16_const1_const2(half %a) {
; GFX1170-TRUE16-LABEL: v_fminimum3_f16_const1_const2:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX1170-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_fminimum3_f16_const1_const2:
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index d321b33ade280..9b755d701ad6c 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -1999,12 +1999,19 @@ define amdgpu_ps half @test_fmin_f16_s_ieee_off(half inreg %a, half inreg %b) #0
; GFX9-NEXT: v_min_f16_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: test_fmin_f16_s_ieee_off:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_min_num_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmin_f16_s_ieee_off:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_min_num_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fmin_f16_s_ieee_off:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_min_num_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.minnum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 79e76039d6777..1ef26e182cb3d 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6088,9 +6088,9 @@ define half @v_contract_mul_add_f16_select_64_1(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_64_1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3c00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6168,9 +6168,9 @@ define half @v_contract_mul_add_f16_select_1_64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_1_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x3c00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x3c00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6248,9 +6248,9 @@ define half @v_contract_mul_add_f16_select_n64_n1(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n64_n1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xbc00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0xd400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xd400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6328,9 +6328,9 @@ define half @v_contract_mul_add_f16_select_n1_n64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n1_n64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xd400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0xbc00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xd400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xbc00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6415,9 +6415,9 @@ define half @v_contract_mul_add_f16_select_128_64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_128_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5800, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6501,9 +6501,9 @@ define half @v_contract_mul_add_f16_select_128_4(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_128_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x4400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5800, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6588,9 +6588,9 @@ define half @v_contract_mul_add_f16_select_2_4(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_2_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x4400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6674,9 +6674,9 @@ define half @v_contract_mul_add_f16_select_4_128(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_4_128:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5800
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x4400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5800
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 1a3bd8b68f964..7c832a71993e1 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -766,27 +766,28 @@ define <2 x half> @select_fneg_select_v2f16(<2 x i1> %cond0, <2 x i1> %cond1, <2
; GFX11-TRUE16-LABEL: select_fneg_select_v2f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v4, 0x80008000, v4
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v4
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v4.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80008000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.l, vcc_lo
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -877,27 +878,28 @@ define <2 x i16> @select_fneg_xor_select_v2i16(<2 x i1> %cond0, <2 x i1> %cond1,
; GFX11-TRUE16-LABEL: select_fneg_xor_select_v2i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v4, 0x80008000, v4
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v4
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v4.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80008000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.l, vcc_lo
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index de0684d853a23..b852d3942e15e 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -1961,7 +1961,8 @@ define <2 x i1> @test_s_signed_v2f64_v2i1(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i1> @llvm.fptosi.sat.v2f64.v2i1(<2 x double> %f)
ret <2 x i1> %x
@@ -2012,38 +2013,75 @@ define <2 x i8> @test_s_signed_v2f64_v2i8(<2 x double> inreg %f) {
; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_signed_v2f64_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
-; GFX11-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
-; GFX11-NEXT: s_movk_i32 s0, 0xff80
-; GFX11-NEXT: v_med3_i32 v2, v0, s0, 0x7f
-; GFX11-NEXT: v_med3_i32 v0, v1, s0, 0x7f
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v2
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_signed_v2f64_v2i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
+; GFX11-FAKE16-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_movk_i32 s0, 0xff80
+; GFX11-FAKE16-NEXT: v_med3_i32 v2, v0, s0, 0x7f
+; GFX11-FAKE16-NEXT: v_med3_i32 v0, v1, s0, 0x7f
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-ISEL-LABEL: test_s_signed_v2f64_v2i8:
-; GFX12-ISEL: ; %bb.0:
-; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-ISEL-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
-; GFX12-ISEL-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
-; GFX12-ISEL-NEXT: s_movk_i32 s0, 0xff80
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: v_med3_i32 v2, v0, s0, 0x7f
-; GFX12-ISEL-NEXT: v_med3_i32 v0, v1, s0, 0x7f
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v2
-; GFX12-ISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-ISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-ISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_signed_v2f64_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
+; GFX11-TRUE16-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_movk_i32 s0, 0xff80
+; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_signed_v2f64_v2i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
+; GFX12-FAKE16-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_movk_i32 s0, 0xff80
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_med3_i32 v2, v0, s0, 0x7f
+; GFX12-FAKE16-NEXT: v_med3_i32 v0, v1, s0, 0x7f
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_signed_v2f64_v2i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f64_e32 v0, s[2:3]
+; GFX12-TRUE16-NEXT: v_cvt_i32_f64_e32 v1, s[0:1]
+; GFX12-TRUE16-NEXT: s_movk_i32 s0, 0xff80
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v2f64_v2i8:
; GFX12-GI: ; %bb.0:
@@ -2062,7 +2100,8 @@ define <2 x i8> @test_s_signed_v2f64_v2i8(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i8> @llvm.fptosi.sat.v2f64.v2i8(<2 x double> %f)
ret <2 x i8> %x
@@ -3377,20 +3416,22 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX11-TRUE16-LABEL: test_s_signed_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s1
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i32 v4, v2, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i32 v2, v1, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i32 v1, v4, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, -1, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_signed_v4f16_v4i1:
@@ -3424,21 +3465,24 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX12-TRUE16-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i32 v4, v2, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i32 v2, v1, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i32 v1, v4, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, -1, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v4f16_v4i1:
@@ -3479,8 +3523,10 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3631,16 +3677,19 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s1, 0x7f
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v0, v1
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v3, s1, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff00, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff00, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_signed_v4f16_v4i8:
@@ -3707,16 +3756,19 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s1, 0x7f
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v0, v1
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v3, s1, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xff00, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xff00, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v4f16_v4i8:
@@ -3757,8 +3809,10 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -4282,23 +4336,23 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v8.l, v4.l, -1, 0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v2.l, v2.l, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v4.l, v0.h, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.l, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v6.l, v2.h, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.h, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v6.l, v4.h, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, -1, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -4505,37 +4559,37 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v3.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v6.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v3.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i8:
@@ -4611,37 +4665,37 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v3.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v6.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
; GFX12-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h
; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v3.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v1.h, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i8:
@@ -4651,24 +4705,24 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_mov_b16_e32 v4.h, 0xff80
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, 0xff80
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v8.l, v4.l, v4.h, 0x7f
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.h, v4.h
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v5.h, v3.l
+; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, v2.h, 0x7f
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v1.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v2.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v3.l
-; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v3.l, v0.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v4.l, v1.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.l, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v6.l, v2.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v4.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v2.l, v2.l, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v3.l, v0.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v4.l, v1.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v5.l, v4.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v6.l, v5.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, v2.h, 0x7f
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -5565,9 +5619,9 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s5
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, s3
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s2
@@ -5577,16 +5631,17 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_med3_i32 v5, v1, -1, 0
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s1
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v6, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX11-TRUE16-NEXT: v_med3_i32 v8, v4, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i32 v6, v4, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i32 v4, v8, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i32 v4, v6, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, -1, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_signed_v8f16_v8i1:
@@ -5644,9 +5699,9 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, s3
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s2
@@ -5657,16 +5712,17 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s1
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX12-TRUE16-NEXT: v_bfe_i32 v8, v6, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX12-TRUE16-NEXT: v_med3_i32 v8, v4, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i32 v6, v4, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i32 v4, v8, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i32 v4, v6, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, -1, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v8f16_v8i1:
@@ -5741,11 +5797,15 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_max_i32 s6, s6, -1
; GFX12-GI-NEXT: s_max_i32 s7, s7, -1
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5983,58 +6043,63 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16-LABEL: test_s_signed_v8f16_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s2
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, s1
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, s4
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v5.l, s3
-; GFX11-TRUE16-NEXT: s_movk_i32 s2, 0xff80
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 16
; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s1
; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 16
+; GFX11-TRUE16-NEXT: s_movk_i32 s2, 0xff80
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v7.l, s0
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, s2, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v4, v4, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 16
; GFX11-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s0
-; GFX11-TRUE16-NEXT: v_med3_i32 v9, v2, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v6, v6, s2, 0x7f
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v4
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v6, 0, 16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v5, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xff00, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_signed_v8f16_v8i8:
@@ -6108,61 +6173,67 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, s4
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, s4
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v5.l, s3
-; GFX12-TRUE16-NEXT: s_movk_i32 s2, 0xff80
; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 16
; GFX12-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s1
; GFX12-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 16
+; GFX12-TRUE16-NEXT: s_movk_i32 s2, 0xff80
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v7.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, s2, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v4, v4, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 16
+; GFX12-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 16
; GFX12-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s0
-; GFX12-TRUE16-NEXT: v_med3_i32 v9, v2, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v6, v6, s2, 0x7f
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v0, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v3, v4
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v6, 0, 16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v9
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v5, v4
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v8
+; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 16
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v6
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xff00, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xff, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v4, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v5
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v7, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v8f16_v8i8:
@@ -6237,11 +6308,15 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_max_i32 s6, s6, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s7, s7, 0xffffff80
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -7188,60 +7263,121 @@ define <4 x i8> @test_s_signed_v4f32_v4i8(<4 x float> inreg %f) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_signed_v4f32_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_i32_f32_e32 v0, s3
-; GFX11-NEXT: v_cvt_i32_f32_e32 v1, s2
-; GFX11-NEXT: s_movk_i32 s2, 0xff80
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, s1
-; GFX11-NEXT: v_cvt_i32_f32_e32 v3, s0
-; GFX11-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX11-NEXT: v_med3_i32 v1, v1, s2, 0x7f
-; GFX11-NEXT: v_med3_i32 v2, v2, s2, 0x7f
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 8, v2
-; GFX11-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX11-NEXT: v_med3_i32 v0, v3, s2, 0x7f
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff00, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_signed_v4f32_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_i32_f32_e32 v0, s3
+; GFX11-FAKE16-NEXT: v_cvt_i32_f32_e32 v1, s2
+; GFX11-FAKE16-NEXT: s_movk_i32 s2, 0xff80
+; GFX11-FAKE16-NEXT: v_cvt_i32_f32_e32 v2, s1
+; GFX11-FAKE16-NEXT: v_cvt_i32_f32_e32 v3, s0
+; GFX11-FAKE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
+; GFX11-FAKE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX11-FAKE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 8, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-FAKE16-NEXT: v_med3_i32 v0, v3, s2, 0x7f
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff00, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-ISEL-LABEL: test_s_signed_v4f32_v4i8:
-; GFX12-ISEL: ; %bb.0:
-; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-ISEL-NEXT: v_mov_b32_e32 v0, 0x7f
-; GFX12-ISEL-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-ISEL-NEXT: s_cvt_i32_f32 s2, s2
-; GFX12-ISEL-NEXT: s_cvt_i32_f32 s1, s1
-; GFX12-ISEL-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: v_med3_i32 v1, 0xffffff80, s3, v0
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-ISEL-NEXT: v_med3_i32 v2, 0xffffff80, s2, v0
-; GFX12-ISEL-NEXT: v_med3_i32 v3, 0xffffff80, s1, v0
-; GFX12-ISEL-NEXT: v_med3_i32 v0, 0xffffff80, s0, v0
-; GFX12-ISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-ISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-ISEL-NEXT: v_or_b32_e32 v2, v2, v1
-; GFX12-ISEL-NEXT: v_and_b32_e32 v1, 0xff00, v3
-; GFX12-ISEL-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-ISEL-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX12-ISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v4
-; GFX12-ISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_signed_v4f32_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, s3
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, s2
+; GFX11-TRUE16-NEXT: s_movk_i32 s2, 0xff80
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, s1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, s0
+; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_med3_i32 v0, v3, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff00, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_signed_v4f32_v4i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7f
+; GFX12-FAKE16-NEXT: s_cvt_i32_f32 s3, s3
+; GFX12-FAKE16-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-FAKE16-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-FAKE16-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_med3_i32 v1, 0xffffff80, s3, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX12-FAKE16-NEXT: v_med3_i32 v2, 0xffffff80, s2, v0
+; GFX12-FAKE16-NEXT: v_med3_i32 v3, 0xffffff80, s1, v0
+; GFX12-FAKE16-NEXT: v_med3_i32 v0, 0xffffff80, s0, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v2, v2, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff00, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_signed_v4f32_v4i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0x7f
+; GFX12-TRUE16-NEXT: s_cvt_i32_f32 s3, s3
+; GFX12-TRUE16-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-TRUE16-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-TRUE16-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_med3_i32 v1, 0xffffff80, s3, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_med3_i32 v2, 0xffffff80, s2, v0
+; GFX12-TRUE16-NEXT: v_med3_i32 v3, 0xffffff80, s1, v0
+; GFX12-TRUE16-NEXT: v_med3_i32 v0, 0xffffff80, s0, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xff00, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v4f32_v4i8:
; GFX12-GI: ; %bb.0:
@@ -7265,8 +7401,10 @@ define <4 x i8> @test_s_signed_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f32.v4i8(<4 x float> %f)
ret <4 x i8> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
index 8b6df1ca1e69c..a58f0cd534968 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
@@ -369,19 +369,47 @@ define i8 @test_s_unsigned_i8_f32(float inreg %f) nounwind {
; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_s_unsigned_i8_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_min_u32 s0, s0, 0xff
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-FAKE16-LABEL: test_s_unsigned_i8_f32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_unsigned_i8_f32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_s_unsigned_i8_f32:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i8 @llvm.fptoui.sat.i8.f32(float %f)
ret i8 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 2b6f2aa2ab6d4..aa29522bbf4ea 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1857,7 +1857,8 @@ define <2 x i1> @test_s_unsigned_v2f64_v2i1(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i1> @llvm.fptoui.sat.v2f64.v2i1(<2 x double> %f)
ret <2 x i1> %x
@@ -1899,31 +1900,59 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX11-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-ISEL-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX12-ISEL: ; %bb.0:
-; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX12-ISEL-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX12-ISEL-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX12-ISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX12-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX12-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v2f64_v2i8:
; GFX12-GI: ; %bb.0:
@@ -1939,7 +1968,8 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i8> @llvm.fptoui.sat.v2f64.v2i8(<2 x double> %f)
ret <2 x i8> %x
@@ -3141,20 +3171,22 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX11-TRUE16-LABEL: test_s_unsigned_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3188,21 +3220,24 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3233,8 +3268,10 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3360,17 +3397,19 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -3426,17 +3465,19 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -3467,8 +3508,10 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3981,23 +4024,23 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_min_u16 v1.l, v0.h, 1
-; GFX12-GI-NEXT: v_min_u16 v8.l, v4.l, 1
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_min_u16 v0.l, v0.l, 1
+; GFX12-GI-NEXT: v_min_u16 v2.l, v2.l, 1
; GFX12-GI-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX12-GI-NEXT: v_min_u16 v4.l, v0.h, 1
-; GFX12-GI-NEXT: v_min_u16 v5.l, v2.l, 1
-; GFX12-GI-NEXT: v_min_u16 v6.l, v2.h, 1
+; GFX12-GI-NEXT: v_min_u16 v5.l, v2.h, 1
+; GFX12-GI-NEXT: v_min_u16 v6.l, v4.h, 1
; GFX12-GI-NEXT: v_min_u16 v7.l, v3.h, 1
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -4178,37 +4221,37 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v5.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v0.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
; GFX11-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v4.l
; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v0.l, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i8:
@@ -4273,37 +4316,37 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v5.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v6.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v0.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v1.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
; GFX12-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v4.l
; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
-; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX12-TRUE16-NEXT: v_or_b16 v4.h, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v1.h
; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v2.l
-; GFX12-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v2.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
-; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v3.l, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_or_b16 v4.l, v3.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i8:
@@ -4313,23 +4356,23 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_min_u16 v1.l, 0xff, v0.h
-; GFX12-GI-NEXT: v_min_u16 v8.l, 0xff, v4.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_min_u16 v0.l, 0xff, v0.l
+; GFX12-GI-NEXT: v_min_u16 v2.l, 0xff, v2.l
; GFX12-GI-NEXT: v_min_u16 v3.l, 0xff, v1.h
; GFX12-GI-NEXT: v_min_u16 v4.l, 0xff, v0.h
-; GFX12-GI-NEXT: v_min_u16 v5.l, 0xff, v2.l
-; GFX12-GI-NEXT: v_min_u16 v6.l, 0xff, v2.h
+; GFX12-GI-NEXT: v_min_u16 v5.l, 0xff, v2.h
+; GFX12-GI-NEXT: v_min_u16 v6.l, 0xff, v4.h
; GFX12-GI-NEXT: v_min_u16 v7.l, 0xff, v3.h
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -5196,9 +5239,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5208,16 +5251,17 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 1, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5275,9 +5319,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5288,16 +5332,17 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5350,11 +5395,15 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s7, s7, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_min_u32 s6, s6, 1
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5560,51 +5609,56 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v4
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v1, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5672,53 +5726,58 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v4
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v1, v4
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5771,11 +5830,15 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s7, s7, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_min_u32 s6, s6, 0xff
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -6649,60 +6712,119 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_unsigned_v4f32_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v1, s2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v2, s1
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX11-NEXT: v_min_u32_e32 v3, 0xff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s3
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v1, s2
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v2, s1
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v3, 0xff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-ISEL-LABEL: test_s_unsigned_v4f32_v4i8:
-; GFX12-ISEL: ; %bb.0:
-; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-ISEL-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-ISEL-NEXT: s_cvt_u32_f32 s2, s2
-; GFX12-ISEL-NEXT: s_cvt_u32_f32 s1, s1
-; GFX12-ISEL-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_min_u32 s3, s3, 0xff
-; GFX12-ISEL-NEXT: s_min_u32 s2, s2, 0xff
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_lshl_b32 s3, s3, 8
-; GFX12-ISEL-NEXT: s_min_u32 s1, s1, 0xff
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_or_b32 s2, s2, s3
-; GFX12-ISEL-NEXT: s_min_u32 s0, s0, 0xff
-; GFX12-ISEL-NEXT: s_lshl_b32 s1, s1, 8
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_lshl_b32 s3, s2, 16
-; GFX12-ISEL-NEXT: s_or_b32 s0, s0, s1
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_or_b32 s1, s1, s3
-; GFX12-ISEL-NEXT: s_lshr_b32 s3, s3, 24
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: s_lshr_b32 s1, s1, 8
-; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-ISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, s2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, s1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s3, s3
+; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_min_u32 s3, s3, 0xff
+; GFX12-FAKE16-NEXT: s_min_u32 s2, s2, 0xff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX12-FAKE16-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s2, s3
+; GFX12-FAKE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s2, 16
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s1, s1, s3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, 24
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s3, s3
+; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_min_u32 s3, s3, 0xff
+; GFX12-TRUE16-NEXT: s_min_u32 s2, s2, 0xff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX12-TRUE16-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s3, s2, 16
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 24
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f32_v4i8:
; GFX12-GI: ; %bb.0:
@@ -6721,8 +6843,10 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f32.v4i8(<4 x float> %f)
ret <4 x i8> %x
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index e36a807a7ceed..d17edcb72f992 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1347,48 +1347,26 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: void_func_v16i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: void_func_v16i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX11-FAKE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: void_func_v16i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
+; GFX11-NEXT: v_lshl_or_b32 v3, v13, 16, v12
+; GFX11-NEXT: v_lshl_or_b32 v2, v9, 16, v8
+; GFX11-NEXT: v_lshl_or_b32 v1, v5, 16, v4
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: v_lshl_or_b32 v0, v6, 16, v0
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
store <16 x i8> %arg0, ptr addrspace(1) null
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index dd9548c42e148..66a27bf6258c4 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -1106,32 +1106,70 @@ define <16 x i8> @v16i8_func_void() #0 {
; GFX789-NEXT: v_mov_b32_e32 v3, v18
; GFX789-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v16i8_func_void:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
-; GFX11-NEXT: v_mov_b32_e32 v2, v17
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v16i8_func_void:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v16i8_func_void:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v17
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) poison
%val = load <16 x i8>, ptr addrspace(1) %ptr
ret <16 x i8> %val
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 65d3608744000..14212cb73727b 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -3692,39 +3692,74 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_v5i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v40, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 16
-; GFX11-NEXT: v_writelane_b32 v40, s30, 0
-; GFX11-NEXT: v_writelane_b32 v40, s31, 1
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_v5i8 at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_v5i8 at abs32@lo
-; GFX11-NEXT: global_load_b64 v[5:6], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX11-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-NEXT: v_mov_b32_e32 v4, v6
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_readlane_b32 s30, v40, 0
-; GFX11-NEXT: v_readlane_b32 s31, v40, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v40, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v5i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v5i8 at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v5i8 at abs32@lo
+; GFX11-TRUE16-NEXT: global_load_b64 v[3:4], v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v5i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v5i8 at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v5i8 at abs32@lo
+; GFX11-FAKE16-NEXT: global_load_b64 v[5:6], v[0:1], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v6
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v5i8:
; GFX10-SCRATCH: ; %bb.0:
@@ -3846,41 +3881,83 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_v8i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v40, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 16
-; GFX11-NEXT: v_writelane_b32 v40, s30, 0
-; GFX11-NEXT: v_writelane_b32 v40, s31, 1
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v8, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v8
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_readlane_b32 s30, v40, 0
-; GFX11-NEXT: v_readlane_b32 s31, v40, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v40, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v8i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v8i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
+; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v8
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v8i8:
; GFX10-SCRATCH: ; %bb.0:
@@ -4070,71 +4147,157 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_v32i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v40, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 16
-; GFX11-NEXT: v_writelane_b32 v40, s30, 0
-; GFX11-NEXT: v_writelane_b32 v40, s31, 1
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-NEXT: global_load_b128 v[16:19], v[4:5], off
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v37, 24, v0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v32, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v34, 24, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v27, 24, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v19
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v35
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_mov_b32_e32 v12, v3
-; GFX11-NEXT: v_mov_b32_e32 v20, v17
-; GFX11-NEXT: v_mov_b32_e32 v24, v18
-; GFX11-NEXT: v_mov_b32_e32 v28, v19
-; GFX11-NEXT: v_dual_mov_b32 v2, v36 :: v_dual_mov_b32 v3, v37
-; GFX11-NEXT: v_dual_mov_b32 v17, v32 :: v_dual_mov_b32 v18, v33
-; GFX11-NEXT: v_mov_b32_e32 v19, v34
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_readlane_b32 s30, v40, 0
-; GFX11-NEXT: v_readlane_b32 s31, v40, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v40, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v32i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[4:5], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v32i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b128 v[16:19], v[4:5], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 24, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v32, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 24, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v27, 24, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v17
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v36 :: v_dual_mov_b32 v3, v37
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v32 :: v_dual_mov_b32 v18, v33
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, v34
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v32i8:
; GFX10-SCRATCH: ; %bb.0:
@@ -4514,10 +4677,9 @@ define amdgpu_gfx void @test_call_external_void_func_v2i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: global_store_b16 v[40:41], v0, off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
@@ -4708,93 +4870,49 @@ define amdgpu_gfx void @test_call_external_void_func_v3i8_ret() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_call_external_void_func_v3i8_ret:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_store_b8 v[3:4], v2, off
-; GFX11-TRUE16-NEXT: global_store_b16 v[40:41], v0, off
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_call_external_void_func_v3i8_ret:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-FAKE16-NEXT: ; meta instruction
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_store_b8 v[3:4], v2, off
-; GFX11-FAKE16-NEXT: global_store_b16 v[40:41], v0, off
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
-; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_call_external_void_func_v3i8_ret:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, s33
+; GFX11-NEXT: s_mov_b32 s33, s32
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: v_writelane_b32 v42, s0, 2
+; GFX11-NEXT: s_add_i32 s32, s32, 16
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GFX11-NEXT: scratch_store_b32 off, v40, s33 offset:4
+; GFX11-NEXT: ; meta instruction
+; GFX11-NEXT: scratch_store_b32 off, v41, s33
+; GFX11-NEXT: v_writelane_b32 v42, s30, 0
+; GFX11-NEXT: v_writelane_b32 v42, s31, 1
+; GFX11-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
+; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
+; GFX11-NEXT: global_load_b32 v0, v[40:41], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-NEXT: v_readlane_b32 s31, v42, 1
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b8 v[3:4], v2, off
+; GFX11-NEXT: global_store_b16 v[40:41], v0, off
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX11-NEXT: scratch_load_b32 v41, off, s33
+; GFX11-NEXT: scratch_load_b32 v40, off, s33 offset:4
+; GFX11-NEXT: s_mov_b32 s32, s33
+; GFX11-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 s33, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v3i8_ret:
; GFX10-SCRATCH: ; %bb.0:
@@ -4918,110 +5036,65 @@ define amdgpu_gfx void @test_call_external_void_func_v4i8_ret() #0 {
; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-NEXT: v_readlane_b32 s30, v42, 0
-; GFX10-NEXT: v_readlane_b32 s31, v42, 1
-; GFX10-NEXT: s_mov_b32 s32, s33
-; GFX10-NEXT: v_readlane_b32 s34, v42, 2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: global_store_dword v[40:41], v0, off
-; GFX10-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX10-NEXT: buffer_load_dword v41, off, s[0:3], s33
-; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:4
-; GFX10-NEXT: s_or_saveexec_b32 s35, -1
-; GFX10-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s35
-; GFX10-NEXT: s_mov_b32 s33, s34
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: test_call_external_void_func_v4i8_ret:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT: global_store_b32 v[40:41], v0, off
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX10-NEXT: v_readlane_b32 s31, v42, 1
+; GFX10-NEXT: s_mov_b32 s32, s33
+; GFX10-NEXT: v_readlane_b32 s34, v42, 2
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: global_store_dword v[40:41], v0, off
+; GFX10-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX10-NEXT: buffer_load_dword v41, off, s[0:3], s33
+; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:4
+; GFX10-NEXT: s_or_saveexec_b32 s35, -1
+; GFX10-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s35
+; GFX10-NEXT: s_mov_b32 s33, s34
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: test_call_external_void_func_v4i8_ret:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-FAKE16-NEXT: ; meta instruction
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
-; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-FAKE16-NEXT: global_store_b32 v[40:41], v0, off
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_call_external_void_func_v4i8_ret:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, s33
+; GFX11-NEXT: s_mov_b32 s33, s32
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: v_writelane_b32 v42, s0, 2
+; GFX11-NEXT: s_add_i32 s32, s32, 16
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GFX11-NEXT: scratch_store_b32 off, v40, s33 offset:4
+; GFX11-NEXT: ; meta instruction
+; GFX11-NEXT: scratch_store_b32 off, v41, s33
+; GFX11-NEXT: v_writelane_b32 v42, s30, 0
+; GFX11-NEXT: v_writelane_b32 v42, s31, 1
+; GFX11-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
+; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
+; GFX11-NEXT: global_load_b32 v0, v[40:41], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX11-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-NEXT: v_readlane_b32 s31, v42, 1
+; GFX11-NEXT: s_mov_b32 s32, s33
+; GFX11-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: global_store_b32 v[40:41], v0, off
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX11-NEXT: scratch_load_b32 v41, off, s33
+; GFX11-NEXT: scratch_load_b32 v40, off, s33 offset:4
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 s33, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v4i8_ret:
; GFX10-SCRATCH: ; %bb.0:
@@ -5190,29 +5263,30 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v5i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v5i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[5:6], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[3:4], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v0, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v2, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b8 v[0:1], v4, off
; GFX11-TRUE16-NEXT: global_store_b32 v[40:41], v2, off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
@@ -5449,31 +5523,35 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8_ret at abs32@lo
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: global_store_b64 v[40:41], v[0:1], off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
@@ -5806,111 +5884,237 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_v32i8_ret:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v44, s33 offset:16 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v44, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 32
-; GFX11-NEXT: s_clause 0x3 ; 16-byte Folded Spill
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 offset:12
-; GFX11-NEXT: ; meta instruction
-; GFX11-NEXT: scratch_store_b32 off, v41, s33 offset:8
-; GFX11-NEXT: ; meta instruction
-; GFX11-NEXT: scratch_store_b32 off, v42, s33 offset:4
-; GFX11-NEXT: ; meta instruction
-; GFX11-NEXT: scratch_store_b32 off, v43, s33
-; GFX11-NEXT: v_writelane_b32 v44, s30, 0
-; GFX11-NEXT: v_writelane_b32 v44, s31, 1
-; GFX11-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-NEXT: v_dual_mov_b32 v42, 16 :: v_dual_mov_b32 v43, 0
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b128 v[0:3], v[40:41], off
-; GFX11-NEXT: global_load_b128 v[16:19], v[42:43], off
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v37, 24, v0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v32, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v34, 24, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v27, 24, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v19
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v35
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_mov_b32_e32 v12, v3
-; GFX11-NEXT: v_mov_b32_e32 v20, v17
-; GFX11-NEXT: v_mov_b32_e32 v24, v18
-; GFX11-NEXT: v_mov_b32_e32 v28, v19
-; GFX11-NEXT: v_dual_mov_b32 v2, v36 :: v_dual_mov_b32 v3, v37
-; GFX11-NEXT: v_dual_mov_b32 v17, v32 :: v_dual_mov_b32 v18, v33
-; GFX11-NEXT: v_mov_b32_e32 v19, v34
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v7, v6, v7, 0xc0c0004
-; GFX11-NEXT: v_lshl_or_b32 v6, v13, 16, v12
-; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-NEXT: v_lshl_or_b32 v5, v9, 16, v8
-; GFX11-NEXT: v_perm_b32 v1, v28, v29, 0xc0c0004
-; GFX11-NEXT: v_lshl_or_b32 v4, v7, 16, v4
-; GFX11-NEXT: v_perm_b32 v7, v30, v31, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v8, v24, v25, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v9, v26, v27, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v11, v20, v21, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v12, v22, v23, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v13, v16, v17, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v14, v18, v19, 0xc0c0004
-; GFX11-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
-; GFX11-NEXT: v_lshl_or_b32 v10, v7, 16, v1
-; GFX11-NEXT: v_lshl_or_b32 v9, v9, 16, v8
-; GFX11-NEXT: v_lshl_or_b32 v8, v12, 16, v11
-; GFX11-NEXT: v_lshl_or_b32 v7, v14, 16, v13
-; GFX11-NEXT: v_lshl_or_b32 v3, v2, 16, v0
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[42:43], v[7:10], off
-; GFX11-NEXT: global_store_b128 v[40:41], v[3:6], off
-; GFX11-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; GFX11-NEXT: scratch_load_b32 v43, off, s33
-; GFX11-NEXT: scratch_load_b32 v42, off, s33 offset:4
-; GFX11-NEXT: scratch_load_b32 v41, off, s33 offset:8
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 offset:12
-; GFX11-NEXT: v_readlane_b32 s30, v44, 0
-; GFX11-NEXT: v_readlane_b32 s31, v44, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v44, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v44, off, s33 offset:16 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v32i8_ret:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s33 offset:16 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v44, s0, 2
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 32
+; GFX11-TRUE16-NEXT: s_clause 0x3 ; 16-byte Folded Spill
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 offset:12
+; GFX11-TRUE16-NEXT: ; meta instruction
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s33 offset:8
+; GFX11-TRUE16-NEXT: ; meta instruction
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s33 offset:4
+; GFX11-TRUE16-NEXT: ; meta instruction
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s33
+; GFX11-TRUE16-NEXT: v_writelane_b32 v44, s30, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v44, s31, 1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v42, 16 :: v_dual_mov_b32 v43, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[42:43], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v6, v7, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v13, 16, v12
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v7, 16, v4
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v24, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v20, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v22, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v16, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v9, 16, v8
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v14, 16, v13
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v2, 16, v0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: global_store_b128 v[42:43], v[7:10], off
+; GFX11-TRUE16-NEXT: global_store_b128 v[40:41], v[3:6], off
+; GFX11-TRUE16-NEXT: s_clause 0x3 ; 16-byte Folded Reload
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s33
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:12
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v44, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v44, 1
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v44, 2
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s33 offset:16 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v32i8_ret:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v44, s33 offset:16 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v44, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 32
+; GFX11-FAKE16-NEXT: s_clause 0x3 ; 16-byte Folded Spill
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 offset:12
+; GFX11-FAKE16-NEXT: ; meta instruction
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s33 offset:8
+; GFX11-FAKE16-NEXT: ; meta instruction
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v42, s33 offset:4
+; GFX11-FAKE16-NEXT: ; meta instruction
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v43, s33
+; GFX11-FAKE16-NEXT: v_writelane_b32 v44, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v44, s31, 1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v42, 16 :: v_dual_mov_b32 v43, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v[40:41], off
+; GFX11-FAKE16-NEXT: global_load_b128 v[16:19], v[42:43], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 24, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v32, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 24, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v27, 24, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v35
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v17
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v24, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v28, v19
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v36 :: v_dual_mov_b32 v3, v37
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v32 :: v_dual_mov_b32 v18, v33
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, v34
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v6, v7, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v13, 16, v12
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v9, 16, v8
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v28, v29, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v7, 16, v4
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v30, v31, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, v24, v25, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v9, v26, v27, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v11, v20, v21, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v22, v23, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v13, v16, v17, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v18, v19, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v7, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v9, 16, v8
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v14, 16, v13
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v2, 16, v0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: global_store_b128 v[42:43], v[7:10], off
+; GFX11-FAKE16-NEXT: global_store_b128 v[40:41], v[3:6], off
+; GFX11-FAKE16-NEXT: s_clause 0x3 ; 16-byte Folded Reload
+; GFX11-FAKE16-NEXT: scratch_load_b32 v43, off, s33
+; GFX11-FAKE16-NEXT: scratch_load_b32 v42, off, s33 offset:4
+; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s33 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 offset:12
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v44, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v44, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v44, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v44, off, s33 offset:16 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v32i8_ret:
; GFX10-SCRATCH: ; %bb.0:
@@ -9437,52 +9641,110 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_v16i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v40, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 16
-; GFX11-NEXT: v_writelane_b32 v40, s30, 0
-; GFX11-NEXT: v_writelane_b32 v40, s31, 1
-; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[0:3], v0, s[0:1]
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
-; GFX11-NEXT: v_mov_b32_e32 v2, v17
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_readlane_b32 s30, v40, 0
-; GFX11-NEXT: v_readlane_b32 s31, v40, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v40, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_v16i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v0, s[0:1]
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_v16i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v0, s[0:1]
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v17
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v16i8:
; GFX10-SCRATCH: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index c1089c70368be..d32896d8571ae 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13717,58 +13717,114 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13813,54 +13869,106 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14154,73 +14262,131 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
@@ -14255,56 +14421,110 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14607,61 +14827,119 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14709,56 +14987,110 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -15058,56 +15390,111 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -15151,71 +15538,122 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
@@ -15441,90 +15879,47 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
}
define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB58_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB58_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB58_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -15924,92 +16319,48 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB59_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB59_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB59_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -16479,32 +16830,33 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16512,7 +16864,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -16597,40 +16949,41 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -16882,66 +17235,35 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4
-; GFX1250-TRUE16-NEXT: v_bfi_b32 v2, 0xffff0000, v3, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off offset:2046
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB61_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off offset:2046
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: .LBB61_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v3, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v3
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB61_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -17305,62 +17627,121 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB62_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -17407,56 +17788,110 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB62_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -17718,90 +18153,47 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB63_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB63_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB63_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB63_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB63_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 2b5feeb686731..3f674bbabca01 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -4898,32 +4898,30 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4931,7 +4929,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5029,40 +5027,38 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5354,32 +5350,30 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5387,7 +5381,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5486,40 +5480,38 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6240,30 +6232,28 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6368,31 +6358,29 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6685,30 +6673,28 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6814,31 +6800,29 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7799,33 +7783,31 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7833,7 +7815,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7932,40 +7914,38 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8262,31 +8242,29 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8392,31 +8370,29 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8701,58 +8677,114 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8797,54 +8829,106 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9096,61 +9180,119 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9197,56 +9339,110 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9506,87 +9702,145 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_movk_i32 s0, 0xf800
-; GFX942-NEXT: s_mov_b32 s1, -1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v5, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_max_f32_e32 v2, v2, v6
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_movk_i32 s0, 0xf800
+; GFX942-NEXT: s_mov_b32 s1, -1
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX942-NEXT: s_movk_i32 s2, 0x7fff
+; GFX942-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX942-NEXT: v_add3_u32 v7, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: s_nop 1
@@ -9608,56 +9862,110 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9917,56 +10225,111 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10010,52 +10373,103 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11108,32 +11522,33 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11141,7 +11556,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11226,40 +11641,41 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11831,62 +12247,121 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11933,56 +12408,110 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 1510e0efeeee2..7c09a40e7c578 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -4898,32 +4898,30 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4931,7 +4929,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5029,40 +5027,38 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5354,32 +5350,30 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5387,7 +5381,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5486,40 +5480,38 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6240,30 +6232,28 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6368,31 +6358,29 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6685,30 +6673,28 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6814,31 +6800,29 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7799,33 +7783,31 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7833,7 +7815,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7932,40 +7914,38 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8262,31 +8242,29 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8392,31 +8370,29 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8701,58 +8677,114 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8797,54 +8829,106 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9096,61 +9180,119 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9197,56 +9339,110 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9506,87 +9702,145 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_movk_i32 s0, 0xf800
-; GFX942-NEXT: s_mov_b32 s1, -1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v5, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_min_f32_e32 v2, v2, v6
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_movk_i32 s0, 0xf800
+; GFX942-NEXT: s_mov_b32 s1, -1
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX942-NEXT: s_movk_i32 s2, 0x7fff
+; GFX942-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX942-NEXT: v_add3_u32 v7, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: s_nop 1
@@ -9608,56 +9862,110 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9917,56 +10225,111 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10010,52 +10373,103 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11108,32 +11522,33 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11141,7 +11556,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11226,40 +11641,41 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11831,62 +12247,121 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11933,56 +12408,110 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index ef314c8f68b1f..f55b03ce5cee7 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9223,58 +9223,114 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9319,54 +9375,106 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9616,61 +9724,119 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9717,56 +9883,110 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10024,61 +10244,119 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10126,56 +10404,110 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10433,56 +10765,111 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
}
define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10526,52 +10913,103 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -11618,32 +12056,33 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11651,7 +12090,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -11736,40 +12175,41 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12337,62 +12777,121 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
}
define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12439,56 +12938,110 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index a1130abf89a7a..dd69d47c7baa5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -2409,11 +2409,17 @@ define amdgpu_ps half @global_load_saddr_f16(ptr addrspace(1) inreg %sbase, i32
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: global_load_saddr_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_u16 v0, v0, s[2:3]
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%load = load half, ptr addrspace(1) %gep0
@@ -2451,11 +2457,17 @@ define amdgpu_ps half @global_load_saddr_f16_immneg128(ptr addrspace(1) inreg %s
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: global_load_saddr_f16_immneg128:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_u16 v0, v0, s[2:3] offset:-128
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_f16_immneg128:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] offset:-128
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_f16_immneg128:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] offset:-128
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
index c3807581d13e1..63f473770d720 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -908,37 +908,37 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1010,42 +1010,43 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -2219,37 +2220,37 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -2321,42 +2322,43 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 0a3874e6f70c0..75f73b0b9a714 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -1010,14 +1010,14 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1048,14 +1048,14 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
@@ -1086,16 +1086,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1136,16 +1136,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1187,16 +1187,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2534,14 +2534,14 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2572,14 +2572,14 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
@@ -2610,16 +2610,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2660,16 +2660,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2711,16 +2711,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -11674,10 +11674,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX11-SDAG-TRUE16-LABEL: multi_use_mul_mad_i16_var:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: multi_use_mul_mad_i16_var:
@@ -11715,10 +11715,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: multi_use_mul_mad_i16_var:
@@ -11782,25 +11782,15 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX1250-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX1250-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX1250-SDAG-REAL16-LABEL: multi_use_mul_mad_i16_var:
-; GFX1250-SDAG-REAL16: ; %bb.0: ; %entry
-; GFX1250-SDAG-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
-; GFX1250-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-REAL16-LABEL: multi_use_mul_mad_i16_var:
-; GFX1250-GISEL-REAL16: ; %bb.0: ; %entry
-; GFX1250-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
-; GFX1250-GISEL-REAL16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v0, v2
-; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-REAL16-LABEL: multi_use_mul_mad_i16_var:
+; GFX1250-REAL16: ; %bb.0: ; %entry
+; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX1250-REAL16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, v2
+; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: multi_use_mul_mad_i16_var:
; GFX1250-GISEL: ; %bb.0: ; %entry
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12552,18 +12542,31 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX11-SDAG-FAKE16-NEXT: v_mad_u64_u32 v[0:1], null, v4, v5, v[2:3]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: mul_u24_known_24bit_add64:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1
-; GFX11-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
-; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: mul_u24_known_24bit_add64:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: mul_u24_known_24bit_add64:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-TRUE16-LABEL: mul_u24_known_24bit_add64:
; GFX1200-SDAG-TRUE16: ; %bb.0:
@@ -12591,23 +12594,41 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1200-SDAG-FAKE16-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1200-GISEL-LABEL: mul_u24_known_24bit_add64:
-; GFX1200-GISEL: ; %bb.0:
-; GFX1200-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1200-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1200-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1
-; GFX1200-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
-; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
-; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1200-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1200-GISEL-TRUE16-LABEL: mul_u24_known_24bit_add64:
+; GFX1200-GISEL-TRUE16: ; %bb.0:
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-TRUE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX1200-GISEL-TRUE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX1200-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1200-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1200-GISEL-FAKE16-LABEL: mul_u24_known_24bit_add64:
+; GFX1200-GISEL-FAKE16: ; %bb.0:
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1200-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-FAKE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX1200-GISEL-FAKE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX1200-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1200-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-SDAG-FAKE16-LABEL: mul_u24_known_24bit_add64:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@@ -12645,8 +12666,8 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1250-GISEL-REAL16: ; %bb.0:
; GFX1250-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-REAL16-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v4, v1.l
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v4
; GFX1250-GISEL-REAL16-NEXT: v_mul_u32_u24_e32 v0, v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
index e99181f425be9..a3ddbdcdcdd50 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
@@ -526,15 +526,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -564,9 +562,9 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -596,15 +594,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -634,9 +630,9 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -666,13 +662,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) {
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY1]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -702,9 +698,9 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) {
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index e3165c1b88093..9e642739ae26f 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -739,14 +739,13 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
; GISEL-GFX11-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-TRUE16-NEXT: {{ $}}
; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY [[COPY3]].lo16
- ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY4]], 0, [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY5]], [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY3]], 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY4]], [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
; GISEL-GFX11-TRUE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX11-FAKE16-LABEL: name: amdgpu_cs_chain_preserve_cc_half
@@ -948,14 +947,13 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
; GISEL-GFX11-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-TRUE16-NEXT: {{ $}}
; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY [[COPY3]].lo16
- ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, [[COPY4]], 0, [[COPY2]], 0, 0, implicit $exec
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY5]], [[V_ADD_NC_U16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i16) into `ptr poison`)
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, [[COPY3]], 0, [[COPY1]], 0, 0, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY4]], [[V_ADD_NC_U16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i16) into `ptr poison`)
; GISEL-GFX11-TRUE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX11-FAKE16-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 76d19d9c6a986..85bd434bed93c 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -1481,8 +1481,8 @@ define half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
- ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
- ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -2219,8 +2219,8 @@ define half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
- ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
- ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index 8a311979dfc1e..eefa2947bcc24 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -326,11 +326,8 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
-; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 18)
%ret_cast = bitcast i16 %ret to half
@@ -393,9 +390,9 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
; GFX1250-SDG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDG-TRUE16-NEXT: v_nop
; GFX1250-SDG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x7d0
+; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x7d0
; GFX1250-SDG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0x3e8 bitop3:0x14
+; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0x3e8 bitop3:0x14
; GFX1250-SDG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_vii:
@@ -485,11 +482,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3e8
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, 0xbb8, v0.l, v0.h bitop3:0x15
-; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, 0xbb8, v0.l, v0.h bitop3:0x15
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_iii:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
index edd9543d861cc..3dd00c7a9d762 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
@@ -645,6 +645,7 @@ define %non_trivial_types @dead_non_trivial(i1 %cond, %non_trivial_types %x, ptr
; ASM-GISEL-TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:84
; ASM-GISEL-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; ASM-GISEL-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; ASM-GISEL-TRUE16-NEXT: ; kill: def $vgpr2_lo16 killed $vgpr2_lo16 def $vgpr2_hi16
; ASM-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; ASM-GISEL-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; ASM-GISEL-TRUE16-NEXT: s_cbranch_execz .LBB3_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
index 0122b70985c86..a49dc122ddc37 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
@@ -162,7 +162,7 @@ define half @v_exp2_f16_uniform(half inreg %src) {
; GFX12-NEXT: v_s_exp_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%exp2 = call half @llvm.amdgcn.exp2.f16(half %src)
ret half %exp2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index 87a41bf3e3cc6..f8ed2b40449c8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -100,11 +100,10 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -168,11 +167,10 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -302,11 +300,10 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -572,12 +569,11 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[1:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -702,10 +698,9 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -902,5 +897,3 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32, float, hal
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 2f288d6fb95a6..8b9b6321d6516 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -304,10 +304,9 @@ main_body:
define amdgpu_ps <4 x float> @load_2dmsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2dmsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -339,11 +338,10 @@ main_body:
define amdgpu_ps <4 x float> @load_2darraymsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2darraymsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[3:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index e3c84dcfcc027..e45eab302d6ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -141,11 +141,10 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -209,11 +208,10 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -343,11 +341,10 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -584,11 +581,10 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1068,12 +1064,11 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[1:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1297,13 +1292,12 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[4:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1366,12 +1360,11 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[8:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1386,13 +1379,12 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12PLUS-TRUE16-LABEL: sample_d_3d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[7:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1572,14 +1564,13 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-TRUE16-LABEL: sample_d_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[4:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[3:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1704,12 +1695,11 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_cl_2d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1792,10 +1782,9 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2104,12 +2093,11 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[7:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2124,13 +2112,12 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2175,12 +2162,11 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[7:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2195,13 +2181,12 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index 3e9c6f4d24d83..d84879762814e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -125,47 +125,19 @@ define amdgpu_ps half @image_sample_2d_f16_tfe(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: global_store_dword v4, v1, s[12:13]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: image_sample_2d_f16_tfe:
-; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_sample v[3:4], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: global_store_b32 v2, v4, s[12:13]
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: image_sample_2d_f16_tfe:
-; GFX11-FAKE16: ; %bb.0: ; %main_body
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: image_sample v[0:1], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: global_store_b32 v4, v1, s[12:13]
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX12PLUS-TRUE16-LABEL: image_sample_2d_f16_tfe:
-; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX12PLUS-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12PLUS-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX12PLUS-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX12PLUS-TRUE16-NEXT: image_sample v[3:4], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX12PLUS-TRUE16-NEXT: global_store_b32 v2, v4, s[12:13]
-; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: image_sample_2d_f16_tfe:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, v1
+; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v5, v4
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-NEXT: image_sample v[0:1], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b32 v4, v1, s[12:13]
+; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: image_sample_2d_f16_tfe:
; GFX12-FAKE16: ; %bb.0: ; %main_body
@@ -638,3 +610,7 @@ declare {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32, fl
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
+; GFX12PLUS-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 8bd42ece040d2..cc3da1ed343df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -511,11 +511,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -530,11 +528,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX13-TRUE16: ; %bb.0: ; %main_body
-; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
@@ -582,11 +578,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -601,11 +595,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX13-TRUE16: ; %bb.0: ; %main_body
-; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index 0f547e619f260..efbe74b6d3c09 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -359,10 +359,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
@@ -410,10 +409,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
index 8efa1133bc8c1..331dbd0add969 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
@@ -427,7 +427,7 @@ define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #
; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-TRUE16-GISEL-NEXT: s_add_f16 s0, s0, s1
; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-GISEL-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-SDAG-LABEL: v_interp_f16_imm_params:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
index d39846a1744c1..f394edd4431ca 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
@@ -693,12 +693,12 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
;
; REAL16-LABEL: name: general_case_load_with_waterfall
; REAL16: bb.0 (%ir-block.0):
- ; REAL16-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; REAL16-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3, $vgpr4, $vgpr5
; REAL16-NEXT: {{ $}}
; REAL16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; REAL16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; REAL16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; REAL16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; REAL16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY $vgpr2_lo16
; REAL16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; REAL16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; REAL16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
index 0756e2142c247..406d70ab1e95b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
@@ -90,13 +90,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(ptr addrspace(8) inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v3f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
index 29e61f1180c73..5b0962338b17a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
@@ -105,20 +105,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(<4 x i32> inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED: ; %bb.0: ; %main_body
-; GFX12-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], null format:[BUF_FMT_32_FLOAT]
-; GFX12-PACKED-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-PACKED-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index 9d5aa73de8324..2a5df3a9739af 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -383,24 +383,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, -1
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_and_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -421,24 +421,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, -1
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_and_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -459,6 +459,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -477,6 +496,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.and.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4019,8 +4057,3 @@ endif:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1132GISEL-FAKE16: {{.*}}
-; GFX1132GISEL-TRUE16: {{.*}}
-; GFX1164GISEL-FAKE16: {{.*}}
-; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index 75a6c7d580573..92539d2670ecd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -383,24 +383,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_or_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -421,24 +421,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_or_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_or_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -459,6 +459,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -477,6 +496,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_or_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.or.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4020,8 +4058,3 @@ endif:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1132GISEL-FAKE16: {{.*}}
-; GFX1132GISEL-TRUE16: {{.*}}
-; GFX1164GISEL-FAKE16: {{.*}}
-; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 5f12243bd0cec..a72575cc47755 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -395,24 +395,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_max_u32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_max_u32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -433,24 +433,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_max_u32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_max_u32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -471,6 +471,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_max_u32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -489,6 +508,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_max_u32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.umax.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4361,8 +4399,3 @@ endif:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1132GISEL-FAKE16: {{.*}}
-; GFX1132GISEL-TRUE16: {{.*}}
-; GFX1164GISEL-FAKE16: {{.*}}
-; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 12357cc9e8726..a97c1213550ec 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -383,24 +383,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, -1
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_min_u32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_min_u32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -421,24 +421,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, -1
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_min_u32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_min_u32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -459,6 +459,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_min_u32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -477,6 +496,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_min_u32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.umin.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4165,8 +4203,3 @@ endif:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1132GISEL-FAKE16: {{.*}}
-; GFX1132GISEL-TRUE16: {{.*}}
-; GFX1164GISEL-FAKE16: {{.*}}
-; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 8e4d7708091b3..b3747240bc224 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -467,24 +467,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_xor_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_xor_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -505,24 +505,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_xor_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_xor_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -543,6 +543,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_xor_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -561,6 +580,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_xor_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.xor.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4805,8 +4843,3 @@ entry:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1132GISEL-FAKE16: {{.*}}
-; GFX1132GISEL-TRUE16: {{.*}}
-; GFX1164GISEL-FAKE16: {{.*}}
-; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
index f5e81c51963ba..9cc25b453e6a5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
@@ -101,14 +101,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(ptr addrspace(8) inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.struct.ptr.tbuffer.load.v3f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
index b98e99c39e7e0..8c4092066fe9a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
@@ -118,22 +118,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(<4 x i32> inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED: ; %bb.0: ; %main_body
-; GFX12-PACKED-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], null format:[BUF_FMT_32_FLOAT] idxen
-; GFX12-PACKED-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-PACKED-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.struct.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
index e17e1c621da2f..5c1fa3a205d04 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
@@ -85,11 +85,10 @@ define amdgpu_ps <3 x float> @sample_gather(<8 x i32> inreg %rsrc, <4 x i32> inr
define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <8 x i32> inreg %rsrc2, i16 %s.16, i16 %t.16, i16 %fragid) {
; GFX11-TRUE16-LABEL: sample_load:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -105,11 +104,10 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX1150-TRUE16-LABEL: sample_load:
; GFX1150-TRUE16: ; %bb.0:
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -159,11 +157,10 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <8 x i32> inreg %rsrc2, i16 %s.16, i16 %t.16, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_sample:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -179,11 +176,10 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX1150-TRUE16-LABEL: load_sample:
; GFX1150-TRUE16: ; %bb.0:
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
index 6ab411182a651..cf5261f3c4053 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
@@ -107,13 +107,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) {
; GFX11-GISEL-TRUE16-LABEL: test_frexp_f16_i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_frexp_f16_i32:
@@ -132,13 +129,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_frexp_f16_i32:
@@ -992,10 +986,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX11-SDAG-TRUE16-LABEL: test_frexp_f16_i16:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1014,10 +1008,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1049,11 +1043,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX11-GISEL-TRUE16-LABEL: test_frexp_f16_i16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1072,11 +1065,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_frexp_f16_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
index c68099fd26c1e..60dba22325d53 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
@@ -1121,14 +1121,16 @@ define <3 x i1> @isnan_v3bf16(<3 x bfloat> %x) nounwind {
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v1
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v3bf16:
@@ -1223,15 +1225,18 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind {
; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l
-; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index 037384edd0906..d6c5c76bae327 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -1431,11 +1431,9 @@ define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 3 :: v_dual_mov_b32 v2, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v1.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v2.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v2f16:
@@ -1568,10 +1566,11 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
-; GFX11SELDAG-TRUE16-NEXT: v_mov_b32_e32 v1, v3
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v3f16:
@@ -1593,14 +1592,11 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 3
; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v4
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v3.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s1, v1.l, v4.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v3f16:
@@ -1754,15 +1750,18 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l
-; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v4.l, v4.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v3.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v4f16:
@@ -1786,17 +1785,13 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 3
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v4, 3 :: v_dual_mov_b32 v5, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.l, v6.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v4
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.h, v7.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v1, v5
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v3.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s1, v1.l, v4.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s2, v1.h, v5.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 1, s2
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
index 17f7a401e96c8..6398e85e5ff44 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
@@ -366,15 +366,25 @@ define <2 x i1> @isnan_v2f32(<2 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v2f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v2f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v2f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <2 x i1> @llvm.is.fpclass.v2f32(<2 x float> %x, i32 3) ; nan
ret <2 x i1> %1
}
@@ -424,18 +434,31 @@ define <3 x i1> @isnan_v3f32(<3 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v3f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v3f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v3f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <3 x i1> @llvm.is.fpclass.v3f32(<3 x float> %x, i32 3) ; nan
ret <3 x i1> %1
}
@@ -493,20 +516,35 @@ define <4 x i1> @isnan_v4f32(<4 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v4f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v4f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v4f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <4 x i1> @llvm.is.fpclass.v4f32(<4 x float> %x, i32 3) ; nan
ret <4 x i1> %1
}
@@ -572,23 +610,38 @@ define <5 x i1> @isnan_v5f32(<5 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v5f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v5f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v5f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <5 x i1> @llvm.is.fpclass.v5f32(<5 x float> %x, i32 3) ; nan
ret <5 x i1> %1
}
@@ -662,25 +715,42 @@ define <6 x i1> @isnan_v6f32(<6 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v6f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v6f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v6f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <6 x i1> @llvm.is.fpclass.v6f32(<6 x float> %x, i32 3) ; nan
ret <6 x i1> %1
}
@@ -762,28 +832,47 @@ define <7 x i1> @isnan_v7f32(<7 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v7f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v7f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v7f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <7 x i1> @llvm.is.fpclass.v7f32(<7 x float> %x, i32 3) ; nan
ret <7 x i1> %1
}
@@ -873,30 +962,51 @@ define <8 x i1> @isnan_v8f32(<8 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v8f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v7, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v8f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v8f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s7, v7, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: v_cndmask_b16 v7.l, 0, 1, s7
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <8 x i1> @llvm.is.fpclass.v8f32(<8 x float> %x, i32 3) ; nan
ret <8 x i1> %1
}
@@ -1050,50 +1160,87 @@ define <16 x i1> @isnan_v16f32(<16 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v15, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v16f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v7, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v8, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v9, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v10, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v11, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v12, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v13, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v14, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v15, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v16f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v8, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v9, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v10, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v11, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v12, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v13, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v14, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v15, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v16f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s7, v7, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s8, v8, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s9, v9, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s10, v10, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s11, v11, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s12, v12, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s13, v13, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s14, v14, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s15, v15, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: v_cndmask_b16 v7.l, 0, 1, s7
+; GFX11GLISEL-NEXT: v_cndmask_b16 v8.l, 0, 1, s8
+; GFX11GLISEL-NEXT: v_cndmask_b16 v9.l, 0, 1, s9
+; GFX11GLISEL-NEXT: v_cndmask_b16 v10.l, 0, 1, s10
+; GFX11GLISEL-NEXT: v_cndmask_b16 v11.l, 0, 1, s11
+; GFX11GLISEL-NEXT: v_cndmask_b16 v12.l, 0, 1, s12
+; GFX11GLISEL-NEXT: v_cndmask_b16 v13.l, 0, 1, s13
+; GFX11GLISEL-NEXT: v_cndmask_b16 v14.l, 0, 1, s14
+; GFX11GLISEL-NEXT: v_cndmask_b16 v15.l, 0, 1, s15
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <16 x i1> @llvm.is.fpclass.v16f32(<16 x float> %x, i32 3) ; nan
ret <16 x i1> %1
}
@@ -1473,15 +1620,25 @@ define <2 x i1> @isnormal_v2f64(<2 x double> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnormal_v2f64:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 0x108
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnormal_v2f64:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 0x108
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnormal_v2f64:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
+; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s1, v[2:3], 0x108
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <2 x i1> @llvm.is.fpclass.v2f64(<2 x double> %x, i32 264) ; 0x108 = "normal"
ret <2 x i1> %1
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index 50430cac9ee3b..db98e27eb852e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -548,9 +548,9 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
; GFX1170-TRUE16-LABEL: v_maximum_f16__nnan_src1:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX1170-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v1.l
+; GFX1170-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_maximum_f16__nnan_src1:
@@ -568,9 +568,9 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximum_f16__nnan_src1:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index 963d4fc819a2a..644600f6fa31a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -476,9 +476,9 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
; GFX1170-TRUE16-LABEL: v_minimum_f16__nnan_src1:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX1170-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v1.l
+; GFX1170-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_minimum_f16__nnan_src1:
@@ -496,9 +496,9 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimum_f16__nnan_src1:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 4e82a8f68366d..16c0058010bc5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -438,13 +438,21 @@ define amdgpu_ps half @sqrt_f16_s(half inreg %x) {
; GFX11-FAKE16-NEXT: v_sqrt_f16_e32 v0, s0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-TRUE16-LABEL: sqrt_f16_s:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-TRUE16-SDAG-LABEL: sqrt_f16_s:
+; GFX12-TRUE16-SDAG: ; %bb.0:
+; GFX12-TRUE16-SDAG-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-TRUE16-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-TRUE16-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-GISEL-LABEL: sqrt_f16_s:
+; GFX12-TRUE16-GISEL: ; %bb.0:
+; GFX12-TRUE16-GISEL-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-GISEL-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: sqrt_f16_s:
; GFX12-FAKE16: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/lround.ll b/llvm/test/CodeGen/AMDGPU/lround.ll
index 887735af85b7d..330f63e64d297 100644
--- a/llvm/test/CodeGen/AMDGPU/lround.ll
+++ b/llvm/test/CodeGen/AMDGPU/lround.ll
@@ -838,16 +838,15 @@ define half @intrinsic_fround_half(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: intrinsic_fround_half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v0.h, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ge_f16_e64 s0, |v1.l|, 0.5
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
+; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: intrinsic_fround_half:
@@ -963,18 +962,17 @@ define i32 @intrinsic_lround_i32_f16(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: intrinsic_lround_i32_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v0.h, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ge_f16_e64 s0, |v1.l|, 0.5
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
+; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
index 10c55a59f84d4..1e7e9b9b05f86 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
@@ -46,22 +46,14 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_constlo(bfloat %src0,
}
define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo(bfloat %src0, bfloat %src1, bfloat %src2, bfloat %lo) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mixhi_bf16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v3
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_fma_mixhi_bf16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_mov_b32_e32 v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
%src1.ext = fpext bfloat %src1 to float
%src2.ext = fpext bfloat %src2 to float
@@ -176,11 +168,10 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_undeflo_clamp_postcvt
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v1, v0, v1, v2 op_sel_hi:[1,1,1]
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, v3, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v3, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX1250-REAL16-NEXT: global_store_b16 v[0:1], v0, off scope:SCOPE_SYS
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GFX1250-REAL16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off scope:SCOPE_SYS
; GFX1250-REAL16-NEXT: s_wait_storecnt 0x0
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 7fd10a09d032d..52b6c4776caba 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -174,20 +174,13 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %s
}
define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src1, half %src2, half %lo) #0 {
-; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; SDAG-GFX11-TRUE16: ; %bb.0:
-; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; SDAG-GFX11-FAKE16: ; %bb.0:
-; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v3
-; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; GFX9: ; %bb.0:
@@ -220,14 +213,6 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; GISEL-GFX11: ; %bb.0:
-; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v3
-; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -611,11 +596,10 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi
; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
; SDAG-GFX11-TRUE16: ; %bb.0:
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v1, v0, v1, v2 op_sel_hi:[1,1,1]
; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v3, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v3, v1, v2 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX11-TRUE16-NEXT: global_store_b16 v[0:1], v0, off dlc
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index f860c5731ee6d..c90d9909ddc4f 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,26 +107,27 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -161,17 +162,19 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -290,8 +293,11 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
@@ -326,8 +332,11 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -12533,26 +12542,27 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -12587,17 +12597,19 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index f1ffce9066d7e..7615f660c98b5 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,26 +109,27 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -163,17 +164,19 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -295,8 +298,11 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
@@ -331,8 +337,11 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -12568,26 +12577,27 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -12622,17 +12632,19 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 121175707a47f..c99f3175d5f7d 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -560,9 +560,8 @@ define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-LABEL: v_fadd_bf16_chain:
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index 8de7a0de592c7..a73b30e4c124d 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -95,13 +95,29 @@ define amdgpu_cs float @v_s_amdgcn_exp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_amdgcn_exp_f16(half inreg %src) {
-; GFX12-LABEL: v_s_amdgcn_exp_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_exp_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_exp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_exp_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_exp_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_amdgcn_exp_f16:
; GCN-GISEL: ; %bb.0:
@@ -210,13 +226,29 @@ define amdgpu_cs float @v_s_amdgcn_log_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_amdgcn_log_f16(half inreg %src) {
-; GFX12-LABEL: v_s_amdgcn_log_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_log_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_amdgcn_log_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_s_amdgcn_log_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_log_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_s_amdgcn_log_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_log_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_amdgcn_log_f16:
; GCN-GISEL: ; %bb.0:
@@ -325,13 +357,29 @@ define amdgpu_cs float @v_s_rcp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_rcp_f16(half inreg %src) {
-; GFX12-LABEL: v_s_rcp_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_rcp_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_rcp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_s_rcp_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_s_rcp_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_rcp_f16:
; GCN-GISEL: ; %bb.0:
@@ -485,13 +533,29 @@ define amdgpu_cs float @v_s_rsq_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_rsq_f16(half inreg %src) {
-; GFX12-LABEL: v_s_rsq_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_rsq_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_rsq_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_s_rsq_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_s_rsq_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_rsq_f16:
; GCN-GISEL: ; %bb.0:
@@ -615,13 +679,29 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_sqrt_f16(half inreg %src) {
-; GFX12-LABEL: v_s_sqrt_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_sqrt_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_s_sqrt_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_s_sqrt_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_sqrt_f16:
; GCN-GISEL: ; %bb.0:
@@ -649,13 +729,29 @@ define amdgpu_cs float @v_amdgcn_sqrt_f32(float inreg %src) {
}
define amdgpu_cs half @v_amdgcn_sqrt_f16(half inreg %src) {
-; GFX12-LABEL: v_amdgcn_sqrt_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_amdgcn_sqrt_f16:
; GCN-GISEL: ; %bb.0:
@@ -888,7 +984,7 @@ define amdgpu_cs half @fdiv_f16_i16(half inreg %a, i16 inreg %b) {
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-GISEL-TRUE16-NEXT: s_mul_f16 s0, s0, s1
; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-FAKE16-LABEL: fdiv_f16_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index f9c85fa324fe1..d54fed408c4c0 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -1366,9 +1366,9 @@ define i16 @v_mul_sub_1_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_1_i16:
@@ -1390,9 +1390,9 @@ define i16 @v_mul_sub_1_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 1
%mul = mul i16 %x, %sub
@@ -1443,9 +1443,9 @@ define i16 @v_mul_sub_1_i16_commute(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.h, v0.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_1_i16_commute:
@@ -1467,9 +1467,9 @@ define i16 @v_mul_sub_1_i16_commute(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.h, v0.l
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 1
%mul = mul i16 %sub, %x
@@ -1597,9 +1597,9 @@ define i16 @v_mul_add_2_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, 2
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, 2
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_add_2_i16:
@@ -1621,9 +1621,9 @@ define i16 @v_mul_add_2_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, 2
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, 2
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%add = add i16 %y, 2
%mul = mul i16 %x, %add
@@ -1674,9 +1674,9 @@ define i16 @v_mul_sub_2_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -2
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -2
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_2_i16:
@@ -1698,9 +1698,9 @@ define i16 @v_mul_sub_2_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -2
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -2
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 2
%mul = mul i16 %x, %sub
@@ -4668,9 +4668,9 @@ define i16 @v_mul_284_add_82_i16(i16 %arg) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.l, 0x11c
+; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, 0x11c
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, 0x52
+; GFX1250-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, 0x52
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_284_add_82_i16:
@@ -4692,9 +4692,9 @@ define i16 @v_mul_284_add_82_i16(i16 %arg) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_mov_b16_e32 v1.l, 0x11c
+; GFX13-REAL16-NEXT: v_mov_b16_e32 v0.h, 0x11c
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, 0x52
+; GFX13-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, 0x52
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%mul = mul i16 %arg, 284
%add = add i16 %mul, 82
diff --git a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
index 8438b8b089cc4..62e160cb955ce 100644
--- a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
+++ b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
@@ -282,10 +282,10 @@ define <2 x half> @v_repeat_divisor_f16_x2_arcp(half %x, half %y, half %D) #0 {
; GFX11-TRUE16-LABEL: v_repeat_divisor_f16_x2_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_f16_x2_arcp:
@@ -569,11 +569,11 @@ define <3 x half> @v_repeat_divisor_f16_x3_arcp(half %x, half %y, half %z, half
; GFX11-TRUE16-LABEL: v_repeat_divisor_f16_x3_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v1.l, v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_f16_x3_arcp:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
index f2fbb1bd4e43d..cd87718634fa1 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
@@ -86,12 +86,26 @@ define amdgpu_vs float @fpext_hif16_to_32(<2 x half> inreg %val) {
}
define amdgpu_vs half @fptrunc_f32_to_f16(float inreg %val) {
-; CHECK-LABEL: fptrunc_f32_to_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_cvt_f16_f32 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: fptrunc_f32_to_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_cvt_f16_f32 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: fptrunc_f32_to_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: fptrunc_f32_to_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = fptrunc float %val to half
ret half %res
}
@@ -151,34 +165,76 @@ define amdgpu_vs float @frint_f32(float inreg %val) {
}
define amdgpu_vs half @fceil_f16(half inreg %val) {
-; CHECK-LABEL: fceil_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_ceil_f16 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: fceil_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_ceil_f16 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: fceil_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_ceil_f16 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: fceil_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_ceil_f16 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.ceil.f16(half %val)
ret half %res
}
define amdgpu_vs half @ffloor_f16(half inreg %val) {
-; CHECK-LABEL: ffloor_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_floor_f16 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: ffloor_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_floor_f16 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: ffloor_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_floor_f16 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: ffloor_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_floor_f16 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.floor.f16(half %val)
ret half %res
}
define amdgpu_vs half @ftrunc_f16(half inreg %val) {
-; CHECK-LABEL: ftrunc_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_trunc_f16 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: ftrunc_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_trunc_f16 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: ftrunc_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_trunc_f16 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: ftrunc_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_trunc_f16 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.trunc.f16(half %val)
ret half %res
}
@@ -227,25 +283,57 @@ define amdgpu_vs i32 @fptoui_f16_to_i32(half inreg %x) {
}
define amdgpu_vs half @sitofp_i32_to_f16(i32 inreg %x) {
-; CHECK-LABEL: sitofp_i32_to_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_cvt_f32_i32 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; CHECK-NEXT: s_cvt_f16_f32 s0, s0
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: sitofp_i32_to_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_cvt_f32_i32 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; SDAG-NEXT: s_cvt_f16_f32 s0, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: sitofp_i32_to_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_cvt_f32_i32 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: sitofp_i32_to_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_cvt_f32_i32 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = sitofp i32 %x to half
ret half %res
}
define amdgpu_vs half @uitofp_i32_to_f16(i32 inreg %x) {
-; CHECK-LABEL: uitofp_i32_to_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_cvt_f32_u32 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; CHECK-NEXT: s_cvt_f16_f32 s0, s0
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: uitofp_i32_to_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_cvt_f32_u32 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; SDAG-NEXT: s_cvt_f16_f32 s0, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: uitofp_i32_to_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_cvt_f32_u32 s0, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: uitofp_i32_to_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_cvt_f32_u32 s0, s0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = uitofp i32 %x to half
ret half %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
index 25d29579583f2..c2103705f0dd8 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
@@ -74,70 +74,26 @@ define amdgpu_vs float @fmax_f32(float inreg %a, float inreg %b) {
}
define amdgpu_vs half @fadd_f16(half inreg %a, half inreg %b) {
-; CHECK-LABEL: fadd_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_add_f16 s0, s0, s1
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
%add = fadd half %a, %b
ret half %add
}
define amdgpu_vs half @fsub_f16(half inreg %a, half inreg %b) {
-; CHECK-LABEL: fsub_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_sub_f16 s0, s0, s1
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
%sub = fsub half %a, %b
ret half %sub
}
define amdgpu_vs half @fmul_f16(half inreg %a, half inreg %b) {
-; CHECK-LABEL: fmul_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mul_f16 s0, s0, s1
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
%mul = fmul half %a, %b
ret half %mul
}
define amdgpu_vs half @fmin_f16(half inreg %a, half inreg %b) {
-; GFX1150-LABEL: fmin_f16:
-; GFX1150: ; %bb.0:
-; GFX1150-NEXT: s_min_f16 s0, s0, s1
-; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: fmin_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_min_num_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
%min = call half @llvm.minnum.f16(half %a, half %b)
ret half %min
}
define amdgpu_vs half @fmax_f16(half inreg %a, half inreg %b) {
-; GFX1150-LABEL: fmax_f16:
-; GFX1150: ; %bb.0:
-; GFX1150-NEXT: s_max_f16 s0, s0, s1
-; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: fmax_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_max_num_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
%max = call half @llvm.maxnum.f16(half %a, half %b)
ret half %max
}
@@ -182,27 +138,12 @@ define amdgpu_vs float @fmac_f32_with_mov(float inreg %a, float inreg %b, float
}
define amdgpu_vs half @fmac_f16(half inreg %a, half inreg %b, half inreg %c) {
-; CHECK-LABEL: fmac_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_fmac_f16 s0, s1, s2
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
%res = call half @llvm.fma.f16(half %b, half %c, half %a)
ret half %res
}
; Check selection of mov + fmac if src2 of fmac has a use later
define amdgpu_vs half @fmac_f16_with_mov(half inreg %a, half inreg %b, half inreg %c) {
-; CHECK-LABEL: fmac_f16_with_mov:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mov_b32 s3, s2
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; CHECK-NEXT: s_fmac_f16 s3, s0, s1
-; CHECK-NEXT: s_add_f16 s0, s3, s2
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
%fma = call half @llvm.fma.f16(half %a, half %b, half %c)
%res = fadd half %fma, %c
ret half %res
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 3bbc5b9113672..8191ad724fc81 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -467,16 +467,12 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX11-LABEL: bitcast_combine_scalar_to_vector_v4i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b16 v1.h, 0xff00, v1.l
-; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
-; GFX11-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: v_and_b16 v0.h, 0xff00, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg.cast = bitcast i16 %arg to <2 x i8>
%tmp1 = shufflevector <2 x i8> %arg.cast, <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
diff --git a/llvm/test/CodeGen/AMDGPU/scmp.ll b/llvm/test/CodeGen/AMDGPU/scmp.ll
index 6ad3fee5344cc..62aa9919bf0ae 100644
--- a/llvm/test/CodeGen/AMDGPU/scmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/scmp.ll
@@ -507,29 +507,29 @@ define <4 x i32> @scmp_v4i8(<4 x i8> %a, <4 x i8> %b) {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-NEXT: v_bfe_i32 v8, v3, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v8, v3, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v4.l
-; GFX12-SDAG-NEXT: v_bfe_i32 v4, v6, 0, 8
+; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v4.l
; GFX12-SDAG-NEXT: v_bfe_i32 v9, v2, 0, 8
; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, v5.l
; GFX12-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v3.l
; GFX12-SDAG-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v3.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v4.l
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v6.l, v7.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v6, vcc_lo
+; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v3, vcc_lo
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v1.l, v2.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v6.l, v7.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v3.l, v4.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
index ef5767738c328..1b71e4a27637d 100644
--- a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
@@ -337,10 +337,10 @@ define half @add_select_fabs_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_fabs_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_sub_f16_e32 v0.l, v1.l, v0.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -387,10 +387,10 @@ define half @add_select_posk_posk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_posk_posk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3c00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0x4000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -436,10 +436,10 @@ define half @add_select_negk_fabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negk_fabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xbc00, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xbc00, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -486,10 +486,10 @@ define half @add_select_negliteralk_fabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negliteralk_fabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xe400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xe400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1021,10 +1021,10 @@ define half @add_select_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1070,10 +1070,10 @@ define half @add_select_negliteralk_negliteralk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negliteralk_negliteralk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xec00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xe800, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xec00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xe800, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1119,10 +1119,10 @@ define half @add_select_fneg_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_fneg_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_sub_f16_e32 v0.l, v1.l, v0.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1672,10 +1672,10 @@ define half @mul_select_posk_negfabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: mul_select_posk_negfabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v1.l, 0x8000, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1774,10 +1774,10 @@ define half @mul_select_negk_negfabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: mul_select_negk_negfabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v1.l, 0x8000, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xc400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xc400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
index 8e7d95e7ce2f5..20e8d8e522d8f 100644
--- a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
+++ b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
@@ -65,8 +65,8 @@ define i64 @srl_bv_not_last(i16 %a, i16 %b, i16 %c, i16 %d) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v1, v3.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v1, v2.h
; GFX1250-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%bv0 = insertelement <4 x i16> poison, i16 %a, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
index e0ebb7ff1927d..10030f4d44601 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
@@ -529,12 +529,19 @@ define amdgpu_ps half @s_constained_fadd_f16_fpexcept_strict(half inreg %x, half
; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e64 v0, s2, s3
; GFX11-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fadd_f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_add_f16 s0, s2, s3
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fadd_f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_f16 s0, s2, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fadd_f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_f16 s0, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.experimental.constrained.fadd.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
index d8925bf2c9a5f..53bb215ea2139 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
@@ -601,12 +601,19 @@ define amdgpu_ps half @s_constained_fmul_f16_fpexcept_strict(half inreg %x, half
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e64 v0, s2, s3
; GFX11-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fmul_f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mul_f16 s0, s2, s3
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fmul_f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_mul_f16 s0, s2, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fmul_f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_mul_f16 s0, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
; GFX1-GISEL-TRUE16-LABEL: s_constained_fmul_f16_fpexcept_strict:
; GFX1-GISEL-TRUE16: ; %bb.0:
; GFX1-GISEL-TRUE16-NEXT: v_mul_f16_e64 v0.l, s2, s3
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 3fa3a3c5a98e0..4d10c2fc523a6 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -761,12 +761,19 @@ define amdgpu_ps half @s_constained_fsub_f16_fpexcept_strict(half inreg %x, half
; GFX11-GISEL-FAKE16-NEXT: v_sub_f16_e64 v0, s2, s3
; GFX11-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fsub_f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_sub_f16 s0, s2, s3
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fsub_f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_sub_f16 s0, s2, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fsub_f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_sub_f16 s0, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index 8ea1c4fa0adc7..fc8b4a1b78694 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -47,15 +47,25 @@ define i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: s_usubsat_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_max_u32 s0, s0, s1
-; GFX11-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: s_usubsat_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
}
@@ -150,15 +160,25 @@ define i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: s_usubsat_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: s_max_u32 s0, s0, s1
-; GFX11-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: s_usubsat_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
ret i16 %result
}
@@ -244,14 +264,23 @@ define i16 @uniform_usubsat_as_bithack_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: uniform_usubsat_as_bithack_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = xor i16 %x, 32768
%result = and i16 %signsplat, %flipsign
@@ -343,14 +372,23 @@ define i16 @uniform_usubsat_as_bithack2_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: uniform_usubsat_as_bithack2_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack2_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack2_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = add i16 %x, 32768
%result = and i16 %signsplat, %flipsign
@@ -442,14 +480,23 @@ define i16 @uniform_usubsat_as_bithack_commute_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: uniform_usubsat_as_bithack_commute_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack_commute_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack_commute_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = add i16 %x, 32768
%result = and i16 %flipsign, %signsplat
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 297d43357c376..c3479d87e5c62 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -8,22 +8,20 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX11-TRUE16-LABEL: swap:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: swap:
@@ -52,15 +50,13 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX12-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
@@ -69,6 +65,7 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: swap:
@@ -116,32 +113,30 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX11-TRUE16-LABEL: swap_B:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB1_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v0.l
; GFX11-TRUE16-NEXT: ;;#ASMEND
-; GFX11-TRUE16-NEXT: v_swap_b16 v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ;;#ASMSTART
-; GFX11-TRUE16-NEXT: ; use v1.l
+; GFX11-TRUE16-NEXT: ; use v0.h
; GFX11-TRUE16-NEXT: ;;#ASMEND
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v2.l
; GFX11-TRUE16-NEXT: ;;#ASMEND
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: swap_B:
@@ -179,25 +174,24 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB1_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: ;;#ASMSTART
; GFX12-TRUE16-NEXT: ; use v0.l
; GFX12-TRUE16-NEXT: ;;#ASMEND
-; GFX12-TRUE16-NEXT: v_swap_b16 v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX12-TRUE16-NEXT: ;;#ASMSTART
-; GFX12-TRUE16-NEXT: ; use v1.l
+; GFX12-TRUE16-NEXT: ; use v0.h
; GFX12-TRUE16-NEXT: ;;#ASMEND
; GFX12-TRUE16-NEXT: ;;#ASMSTART
; GFX12-TRUE16-NEXT: ; use v2.l
; GFX12-TRUE16-NEXT: ;;#ASMEND
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -205,7 +199,7 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: swap_B:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 01b7293dcd7ab..12193c153dd18 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -915,22 +915,22 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v2.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -1029,22 +1029,22 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v2.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index 0ae4b05aa7c89..c61cc1755bfad 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -813,23 +813,23 @@ define i8 @test_vector_reduce_and_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -890,23 +890,23 @@ define i8 @test_vector_reduce_and_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
index e95ba43686e3a..6270b86590caf 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
@@ -394,11 +394,11 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fadd_v4half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
@@ -450,11 +450,11 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
@@ -640,11 +640,11 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fadd_v8half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
@@ -728,11 +728,11 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
@@ -1046,11 +1046,11 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fadd_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v9.l
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
@@ -1198,11 +1198,11 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v9.l
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
index 6291173230114..88cea06e3f019 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
@@ -394,11 +394,11 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmul_v4half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
@@ -450,11 +450,11 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
@@ -640,11 +640,11 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmul_v8half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
@@ -728,11 +728,11 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
@@ -1046,11 +1046,11 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmul_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v9.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
@@ -1198,11 +1198,11 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v9.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index b931b312ff251..1a59b727cf2d0 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -902,23 +902,23 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1004,23 +1004,23 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index d7db9c845b4ad..9e3e273d4253b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -841,23 +841,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -915,23 +915,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 219961c35071d..bb95fce4c5b74 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -208,16 +208,11 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -249,16 +244,11 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -442,11 +432,11 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -510,11 +500,11 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -776,31 +766,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -879,31 +865,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1254,50 +1236,42 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -1405,50 +1379,42 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index 376a64bb86271..c51bda6f5540f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -208,16 +208,11 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -249,16 +244,11 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -442,11 +432,11 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -510,11 +500,11 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -776,31 +766,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -879,31 +865,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1254,50 +1236,42 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -1405,50 +1379,42 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 087832601598a..486f4490d563b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -412,15 +412,15 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
@@ -477,15 +477,15 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
@@ -727,23 +727,23 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
@@ -816,23 +816,23 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
@@ -1181,34 +1181,34 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v0.h, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v3.h, v6.h, v3.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v3.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v6.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.h, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v6.l, v7.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v3.l, v3.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v4.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.l, v2.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v2.h
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -1314,34 +1314,34 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v0.h, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v3.h, v6.h, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v3.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v6.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.h, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v6.l, v7.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v3.l, v3.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v4.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.l, v2.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v2.h
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 5443cce424a5c..2e40022d8fde6 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -367,15 +367,15 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
@@ -419,15 +419,15 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
@@ -613,23 +613,23 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
@@ -688,23 +688,23 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
@@ -996,34 +996,34 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v0.h, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v3.h, v6.h, v3.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v3.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v6.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.h, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v6.l, v7.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v3.l, v3.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v4.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.l, v2.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v2.h
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -1117,34 +1117,34 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v0.h, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v3.h, v6.h, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v3.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v6.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.h, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v6.l, v7.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v3.l, v3.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v4.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.l, v2.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v2.h
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index b9a8ea279ad15..c601ed4c3af81 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -794,23 +794,23 @@ define i8 @test_vector_reduce_xor_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -865,23 +865,23 @@ define i8 @test_vector_reduce_xor_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
index fca3b5176ce58..48809ecc60633 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
@@ -28,9 +28,9 @@ define <2 x i8> @shuffle_v2i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2i8_rebroadcast:
@@ -76,11 +76,11 @@ define <4 x i8> @shuffle_v4i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4i8_rebroadcast:
@@ -136,15 +136,15 @@ define <8 x i8> @shuffle_v8i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v8i8_rebroadcast:
@@ -220,23 +220,23 @@ define <16 x i8> @shuffle_v16i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v16i8_rebroadcast:
@@ -352,39 +352,39 @@ define <32 x i8> @shuffle_v32i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v32i8_rebroadcast:
More information about the llvm-branch-commits
mailing list